Main_py.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.Preparing the Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os                                                   # set current working directory \n",
    "os.chdir(\"C:/Users/hj6488/Box Sync/04_Course work (Spring 2019)/800 ML example/Classifier_Res\")  # set the working directory\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports for array-handling and plotting\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "#matplotlib.use('agg')\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# # let's keep our keras backend tensorflow quiet\n",
    "# import os\n",
    "# os.environ['TF_CPP_MIN_LOG_LEVEL']='3'\n",
    "# # for testing on CPU\n",
    "# #os.environ['CUDA_VISIBLE_DEVICES'] = ''\n",
    "\n",
    "# keras imports for the dataset and building our neural network\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = np.loadtxt('circle')\n",
    "B = np.loadtxt('lobate_0')\n",
    "C = np.loadtxt('lobate_45')\n",
    "D = np.loadtxt('lobate_90')\n",
    "E = np.loadtxt('lobate_135')\n",
    "F = np.loadtxt('Channel_0')\n",
    "G = np.loadtxt('channel_45')\n",
    "H = np.loadtxt('Channel_90')\n",
    "I = np.loadtxt('Channel_135')\n",
    "\n",
    "\n",
    "A_l = np.ones((1,A.shape[1]))*0\n",
    "B_l = np.ones((1,B.shape[1]))*1\n",
    "C_l = np.ones((1,C.shape[1]))*2\n",
    "D_l = np.ones((1,D.shape[1]))*3\n",
    "E_l = np.ones((1,E.shape[1]))*4\n",
    "F_l = np.ones((1,F.shape[1]))*5\n",
    "G_l = np.ones((1,G.shape[1]))*6\n",
    "H_l = np.ones((1,H.shape[1]))*7\n",
    "I_l = np.ones((1,F.shape[1]))*8\n",
    "\n",
    "\n",
    "X_train = np.concatenate((A,B,C,D,E,F,G,H,I),axis=1)\n",
    "y_train = np.concatenate((A_l, B_l, C_l, D_l, E_l, F_l, G_l, H_l, I_l),axis=1)\n",
    "\n",
    "np.random.seed( seed = 1223 )\n",
    "mask_test =   np.random.randint( 0,899, int( X_train.shape[1]*0.2 ) )    #shuffle data\n",
    "mask_test = np.unique(mask_test)\n",
    "#np.random.shuffle( mask_test )\n",
    "\n",
    "X_test = X_train[ :, mask_test ]\n",
    "y_test = y_train[ :, mask_test ]\n",
    "\n",
    "X_train = np.delete(X_train,mask_test,1)\n",
    "y_train = np.delete(y_train,mask_test,1)\n",
    "\n",
    "size_training = X_train.shape[1]\n",
    "size_test = X_test.shape[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784, 731)"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(784, 100)\n",
      "(28, 28, 100)\n"
     ]
    }
   ],
   "source": [
    "print(A.shape)\n",
    "A=A.reshape(28,28,-1)\n",
    "B=B.reshape(28,28,-1)\n",
    "C=C.reshape(28,28,-1)\n",
    "D=D.reshape(28,28,-1)\n",
    "E=E.reshape(28,28,-1)\n",
    "F=F.reshape(28,28,-1)\n",
    "G=G.reshape(28,28,-1)\n",
    "H=H.reshape(28,28,-1)\n",
    "I=I.reshape(28,28,-1)\n",
    "print(A.shape)\n",
    "#plt.figsave('result')\n",
    "# plt.imshow(H[:,:,40])\n",
    "# plt.savefig('result')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hj6488\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\matplotlib\\pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(A[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_Circle')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(B[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_Lobate_0')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(C[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_Lobate_45')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(D[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_Lobate_90')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(E[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_Lobate_135')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(F[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_channel_0')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(G[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_channel_45')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(H[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_channel_90')\n",
    "\n",
    "fig = plt.figure()\n",
    "for i in range(9):\n",
    "  plt.subplot(3,3,i+1)\n",
    "  plt.tight_layout()\n",
    "  plt.imshow(I[:,:,i], interpolation='none')\n",
    "#   plt.title(\"Digit: {}\".format(y_train[i]))\n",
    "  plt.xticks([])\n",
    "  plt.yticks([])\n",
    "fig.savefig('1_Training 3x3_channel_135')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Name_list=['Circle','Lobate_0','Lobate_45','Lobate_90','Lobate_135','channel_0','channel_45','channel_90','channel_135']\n",
    "type(Name_list[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape data\n",
    "X_train = X_train.reshape(28,28,size_training)\n",
    "y_train = y_train.reshape((size_training,))\n",
    "X_test = X_test.reshape(28,28,size_test)\n",
    "y_test = y_test.reshape((size_test,))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hj6488\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\matplotlib\\pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "plt.subplot(2,1,1)\n",
    "plt.imshow(X_train[:,:,0], interpolation='none')\n",
    "plt.title(\"TI type: %s\" %Name_list[int(y_train.T[0])])\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.subplot(2,1,2)\n",
    "plt.hist(X_train[:,:,0].reshape(784))\n",
    "plt.title(\"Pixel Value Distribution\")\n",
    "fig.savefig('2_Training 1x1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape (731, 28, 28)\n",
      "y_train shape (731,)\n",
      "X_test shape (169, 28, 28)\n",
      "y_test shape (169,)\n",
      "Train matrix shape (731, 784)\n",
      "Test matrix shape (169, 784)\n"
     ]
    }
   ],
   "source": [
    "X_train = np.moveaxis(X_train, 2,0)\n",
    "X_test= np.moveaxis(X_test, 2,0)\n",
    "\n",
    "# let's print the shape before we reshape and normalize\n",
    "print(\"X_train shape\", X_train.shape)\n",
    "print(\"y_train shape\", y_train.shape)\n",
    "print(\"X_test shape\", X_test.shape)\n",
    "print(\"y_test shape\", y_test.shape)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# building the input vector from the 28x28 pixels\n",
    "X_train = X_train.reshape(size_training, 784)\n",
    "X_test = X_test.reshape(size_test, 784)\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "# # normalizing the data to help with the training\n",
    "# X_train /= 255\n",
    "# X_test /= 255\n",
    "\n",
    "# print the final input shape ready for training\n",
    "print(\"Train matrix shape\", X_train.shape)\n",
    "print(\"Test matrix shape\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(array([0., 1., 2., 3., 4., 5., 6., 7., 8.]), array([83, 86, 79, 78, 85, 81, 77, 80, 82], dtype=int64))\n"
     ]
    }
   ],
   "source": [
    "print(np.unique(y_train, return_counts=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape before one-hot encoding:  (731,)\n",
      "Shape after one-hot encoding:  (731, 9)\n"
     ]
    }
   ],
   "source": [
    "# For visualization later\n",
    "y_test_=y_test\n",
    "y_train_=y_train\n",
    "\n",
    "# one-hot encoding using keras' numpy-related utilities\n",
    "n_classes = 9\n",
    "print(\"Shape before one-hot encoding: \", y_train.shape)\n",
    "y_train = np_utils.to_categorical(y_train, n_classes)\n",
    "y_test = np_utils.to_categorical(y_test, n_classes)\n",
    "print(\"Shape after one-hot encoding: \", y_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Build Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# building a linear stack of layers with the sequential model\n",
    "model = Sequential()\n",
    "model.add(Dense(512, input_shape=(784,)))\n",
    "model.add(Activation('relu'))                            \n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(9))\n",
    "model.add(Activation('softmax'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Compiling and Training the Model    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# compiling the sequential model\n",
    "model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 731 samples, validate on 169 samples\n",
      "Epoch 1/20\n",
      " - 1s - loss: 2.3440 - acc: 0.1395 - val_loss: 2.1406 - val_acc: 0.2012\n",
      "Epoch 2/20\n",
      " - 0s - loss: 1.9510 - acc: 0.2709 - val_loss: 2.0599 - val_acc: 0.2071\n",
      "Epoch 3/20\n",
      " - 0s - loss: 1.6048 - acc: 0.4145 - val_loss: 1.8544 - val_acc: 0.2189\n",
      "Epoch 4/20\n",
      " - 0s - loss: 1.3095 - acc: 0.5007 - val_loss: 1.7015 - val_acc: 0.3432\n",
      "Epoch 5/20\n",
      " - 0s - loss: 0.9694 - acc: 0.6293 - val_loss: 1.5928 - val_acc: 0.4201\n",
      "Epoch 6/20\n",
      " - 0s - loss: 0.7456 - acc: 0.7360 - val_loss: 1.6505 - val_acc: 0.3432\n",
      "Epoch 7/20\n",
      " - 0s - loss: 0.6028 - acc: 0.7702 - val_loss: 1.7008 - val_acc: 0.3787\n",
      "Epoch 8/20\n",
      " - 0s - loss: 0.4651 - acc: 0.8098 - val_loss: 2.0552 - val_acc: 0.2781\n",
      "Epoch 9/20\n",
      " - 0s - loss: 0.5165 - acc: 0.7811 - val_loss: 1.7247 - val_acc: 0.3787\n",
      "Epoch 10/20\n",
      " - 0s - loss: 0.3653 - acc: 0.8386 - val_loss: 1.6441 - val_acc: 0.3905\n",
      "Epoch 11/20\n",
      " - 0s - loss: 0.2873 - acc: 0.8687 - val_loss: 1.7143 - val_acc: 0.3728\n",
      "Epoch 12/20\n",
      " - 0s - loss: 0.2392 - acc: 0.8824 - val_loss: 1.8034 - val_acc: 0.4083\n",
      "Epoch 13/20\n",
      " - 0s - loss: 0.2340 - acc: 0.8824 - val_loss: 1.9470 - val_acc: 0.3846\n",
      "Epoch 14/20\n",
      " - 0s - loss: 0.2844 - acc: 0.8769 - val_loss: 1.8429 - val_acc: 0.3254\n",
      "Epoch 15/20\n",
      " - 0s - loss: 0.2178 - acc: 0.8892 - val_loss: 1.8392 - val_acc: 0.3728\n",
      "Epoch 16/20\n",
      " - 0s - loss: 0.2142 - acc: 0.8741 - val_loss: 1.7812 - val_acc: 0.3846\n",
      "Epoch 17/20\n",
      " - 0s - loss: 0.2762 - acc: 0.8687 - val_loss: 1.6508 - val_acc: 0.4497\n",
      "Epoch 18/20\n",
      " - 0s - loss: 0.2369 - acc: 0.8755 - val_loss: 2.0063 - val_acc: 0.3669\n",
      "Epoch 19/20\n",
      " - 0s - loss: 0.2373 - acc: 0.8796 - val_loss: 1.8549 - val_acc: 0.3964\n",
      "Epoch 20/20\n",
      " - 0s - loss: 0.2256 - acc: 0.8632 - val_loss: 1.8440 - val_acc: 0.3846\n"
     ]
    }
   ],
   "source": [
    "# training the model and saving metrics in history\n",
    "history = model.fit(X_train, y_train,\n",
    "          batch_size=10, epochs=20,\n",
    "          verbose=2,\n",
    "          validation_data=(X_test, y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved trained model at C:/Users/hj6488/Box Sync/04_Course work (Spring 2019)/800 ML example/Classifier_Res/Results\\keras_MPS.h5 \n"
     ]
    }
   ],
   "source": [
    "# saving the model\n",
    "save_dir = \"C:/Users/hj6488/Box Sync/04_Course work (Spring 2019)/800 ML example/Classifier_Res/Results\"\n",
    "model_name = 'keras_MPS.h5'\n",
    "model_path = os.path.join(save_dir, model_name)\n",
    "model.save(model_path)\n",
    "print('Saved trained model at %s ' % model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hj6488\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\matplotlib\\pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# plotting the metrics\n",
    "fig = plt.figure()\n",
    "plt.subplot(2,1,1)\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='lower right')\n",
    "\n",
    "plt.subplot(2,1,2)\n",
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper right')\n",
    "\n",
    "plt.tight_layout()\n",
    "\n",
    "fig.savefig('3_Loss_and_acc_history')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Evaluate the Model's Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss 1.8439622778864302\n",
      "Test Accuracy 0.3846153860261454\n"
     ]
    }
   ],
   "source": [
    "MPS_model = load_model(\"C:/Users/hj6488/Box Sync/04_Course work (Spring 2019)/800 ML example/Classifier_Res/results/keras_MPS.h5\")\n",
    "loss_and_metrics = MPS_model.evaluate(X_test, y_test, verbose=2)\n",
    "\n",
    "print(\"Test Loss\", loss_and_metrics[0])\n",
    "print(\"Test Accuracy\", loss_and_metrics[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "65  classified correctly\n",
      "104  classified incorrectly\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hj6488\\AppData\\Local\\conda\\conda\\envs\\tensorflow\\lib\\site-packages\\matplotlib\\pyplot.py:514: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  max_open_warning, RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "# load the model and create predictions on the test set\n",
    "MPS_model = load_model(\"C:/Users/hj6488/Box Sync/04_Course work (Spring 2019)/800 ML example/Classifier_Res/results/keras_MPS.h5\")\n",
    "predicted_classes = MPS_model.predict_classes(X_test)\n",
    "\n",
    "# see which we predicted correctly and which not\n",
    "correct_indices = np.nonzero(predicted_classes == y_test_)[0]\n",
    "incorrect_indices = np.nonzero(predicted_classes != y_test_)[0]\n",
    "print()\n",
    "print(len(correct_indices),\" classified correctly\")\n",
    "print(len(incorrect_indices),\" classified incorrectly\")\n",
    "\n",
    "# adapt figure size to accomodate 18 subplots\n",
    "plt.rcParams['figure.figsize'] = (10,20)\n",
    "\n",
    "figure_evaluation = plt.figure()\n",
    "\n",
    "# plot 9 correct predictions\n",
    "for i, correct in enumerate(correct_indices[:9]):\n",
    "    plt.subplot(6,3,i+1)\n",
    "    plt.imshow(X_test[correct].reshape(28,28), interpolation='none')\n",
    "    plt.title(\n",
    "      \"Predicted: %s, Truth: %s\" %(Name_list[int(predicted_classes[correct])],\n",
    "                                        Name_list[int(y_test_[correct])]),fontsize=10)\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "\n",
    "# plot 9 incorrect predictions\n",
    "for i, incorrect in enumerate(incorrect_indices[:9]):\n",
    "    plt.subplot(6,3,i+10)\n",
    "    plt.imshow(X_test[incorrect].reshape(28,28), interpolation='none')\n",
    "    plt.title(\n",
    "      \"Predicted %s, Truth: %s\" %(Name_list[int(predicted_classes[incorrect])],\n",
    "                                        Name_list[int(y_test_[incorrect])]),fontsize=10)\n",
    "    plt.xticks([])\n",
    "    plt.yticks([])\n",
    "\n",
    "figure_evaluation.savefig('4. Incorrect result.png',format = 'png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(169, 9)"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}