{"cells":[{"metadata":{"_uuid":"e34c2c5868997df4ecf1669d59940b2c56460a2d"},"cell_type":"markdown","source":"# Achieving 99% on MNIST\n\nIn this notebook the MNIST dataset is classified by means of a convolutional network with Keras, achieving 99% on the testing set.\n\nTopics include data augmentation and error analysis."},{"metadata":{"_uuid":"1f4f13906a3d737af3bc7ce6e9bb8c4a8695c725"},"cell_type":"markdown","source":"\n\n---"},{"metadata":{"toc":true,"_uuid":"a6407a47692b011184c4b309ddce239b3648b89c"},"cell_type":"markdown","source":"

Content

\n
"},{"metadata":{"trusted":true,"_uuid":"717e2d468cdfd41f1166afd915f2e357da7523f9"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport tensorflow as tf\nfrom tensorflow.python.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.python.keras.models import Sequential, load_model\nfrom tensorflow.python.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, Concatenate\nfrom tensorflow.python.keras.applications.vgg16 import VGG16\nfrom tensorflow.python.keras import optimizers\n\nimport plotly.offline as py\nimport plotly.graph_objs as go\npy.init_notebook_mode(connected=True)\n\nimport matplotlib.pyplot as plt\nimport matplotlib.cm as cm\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51156024bafa089c9b73be68e7fe32a545c80b64"},"cell_type":"code","source":"# CONSTANTS\nSIZE = 28\nLABELS = 10\nCHANNELS = 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7e8e1236f161082fda43eabe3eac1e4be964af2"},"cell_type":"markdown","source":"---\n\n## Visualizer Methods\nThese are some helper functions."},{"metadata":{"trusted":true,"_uuid":"09fb4542e1a928c9e24942d55e81a76a3efcf648"},"cell_type":"code","source":"def digits(data, labels=None, random=False, xy=None):\n if xy is None: x, y = 20, min(10, len(data) // 20 +1)\n else: x, y = xy\n fig, ax = plt.subplots(y, x, figsize = (x, y))\n if x==1: indeces = np.arange(y)\n elif y==1: indeces = np.arange(x)\n else: indeces = [(i,j) for i in np.arange(y) for j in np.arange(x)]\n for i, index in enumerate(indeces[:len(data)]):\n if random: i = np.random.randint(0, len(data))\n ax[index].matshow(data.reshape(-1, SIZE, SIZE)[i], cmap=cm.gray_r)\n if labels: ax[index].set_title(\"Label: {}\".format(labels[i]))\n for index in indeces: ax[index].axis('off')\n plt.subplots_adjust(wspace=0, hspace=0)\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df5bce71fe699f5ff4bb024e7b5406a5eccddd5b"},"cell_type":"code","source":"def plotter(history):\n at, av, lt, lv = zip(*history)\n fig = plt.figure(figsize=(15, 8)); ax1 = fig.add_subplot(221); ax2 = fig.add_subplot(222)\n\n ax1.plot(np.arange(0, len(at), 1), at,\".-\", color='#2A6EA6', label=\"Training: {0:.2f}%\".format(at[-1]))\n ax1.plot(np.arange(0, len(av), 1), av,\".-\", color='#FFA933', label=\"Validation: {0:.2f}%\".format(av[-1]))\n ax1.grid(True); ax1.legend(loc=\"lower right\"); ax1.set_title(\"Accuracy per epoch\")\n\n ax2.plot(np.arange(0, len(lt), 1), lt,\".-\", color='#2A6EA6', label=\"Training: {0:.2f}\".format(lt[-1]))\n ax2.plot(np.arange(0, len(lv), 1), lv,\".-\", color='#FFA933', label=\"Validation: {0:.2f}\".format(lv[-1]))\n ax2.grid(True); ax2.legend(loc=\"upper right\"); ax2.set_title(\"Cost per epoch\")\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2984954ca00f4d40393523ffe7368ff7699b7c86"},"cell_type":"code","source":"def plotCertainty(data, predictions, probabilities):\n fig1, ax1 = plt.subplots(1, 10, figsize = (20, 2))\n fig2, ax2 = plt.subplots(1, 10, figsize = (20, 2))\n for t in np.arange(10):\n ax1[t].matshow(data.reshape(-1, SIZE, SIZE)[t], cmap=cm.gray_r)\n ax1[t].get_yaxis().set_visible(False); ax1[t].get_xaxis().set_visible(False)\n ax2[t].set_title(\"Prediction: {}\".format(predictions[t]))\n ax2[t].plot(probabilities[t], lw=2)\n ax2[t].set_title(\"Prediction: {}\".format(predictions[t]))\n ax2[t].get_yaxis().set_visible(False); ax2[t].set_ylim([0, 1]); ax2[t].grid(True)\n plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2aebfeebf9c574d2dcbf857bdcfbee435f3fbc4"},"cell_type":"markdown","source":"---\n## Data Import\nFirst, we need to import the data."},{"metadata":{"trusted":true,"_uuid":"f87c8d6a22e9db086d60d2caf4d21e2ff5198147"},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/train.csv')\ntest = pd.read_csv('/kaggle/input/test.csv')\nlabels = data.pop('label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22c1b6372d99981af6fc1b95f7a79c4a5a7abbf0"},"cell_type":"code","source":"data.tail()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"37695231c30ec1cf1c45cd7529123bd4e5e6f42f"},"cell_type":"code","source":"labels.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9195a6701e1d683e970698d55f6fe331e409af3e"},"cell_type":"markdown","source":"This function prints some digits."},{"metadata":{"trusted":true,"_uuid":"b667e57a8224dd32a867497e8c6ba6373da8e3ce"},"cell_type":"code","source":"digits(data.values, random=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e89039cd2f5bd331809ddc65792938094384d644"},"cell_type":"markdown","source":"---\n## Data Preproccessing\nThe next step is to reformat the data into a TensorFlow-friendly shape.\n- Convolutions need the image data formatted as a cube (width by height by #channels)\n- Labels need to be 1-hot encodings."},{"metadata":{"trusted":true,"_uuid":"90e3165af0f35b1ecd1dd8395c12bd8241300db2"},"cell_type":"code","source":"print('Training set', data.shape, labels.shape)\nprint('Testing set', test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4a075c8ffdb3f6577cf476fd76929382d4c1d57"},"cell_type":"code","source":"data = data.values.reshape(-1, SIZE, SIZE, CHANNELS)\ntestX = test.values.reshape(-1, SIZE, SIZE, CHANNELS)\nlabels = pd.get_dummies(labels).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a875ee5190378b2bc56ba7d3addffe837e4bac8"},"cell_type":"code","source":"print('Training set', data.shape, labels.shape)\nprint('Testing set', testX.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e7617b300ed149597d522c28d2624331d967e09"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrainX, validX, trainY, validY = train_test_split(data, labels, train_size=32000, random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07b8b1e6db107e187ba580b676618850c374b754"},"cell_type":"code","source":"print('Training set:', trainX.shape, trainY.shape)\nprint('Validation set:', validX.shape, validY.shape)\nprint('Testing set:', testX.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"469c521550160fc896a02f7b7b2f982ae7c7bd56"},"cell_type":"markdown","source":"---\n\n## Conv Net with Keras"},{"metadata":{"_uuid":"80d45b3edae7da0f8c3499790588f40730c4b86a"},"cell_type":"markdown","source":"### Data Generator\nThis generator performs slight transformations on the digits, augmenting our data."},{"metadata":{"trusted":true,"_uuid":"c26a12d1adb3542edb5c3ee7c66e83988d5fcede"},"cell_type":"code","source":"trainAugmenter = ImageDataGenerator(\n rotation_range=5,\n width_shift_range=0.08,\n height_shift_range=0.08,\n shear_range=0.08,\n zoom_range=(0.92, 1.08),\n horizontal_flip=False)\nvalidAugmenter = ImageDataGenerator()\ntestAugmenter = ImageDataGenerator()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f677e91facdefe7bd2dcf75e310e26849c542bb6"},"cell_type":"code","source":"BATCHSIZE = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"085fc06f33a0868279e274a4643ba8ee160a8667"},"cell_type":"code","source":"trainGenerator = trainAugmenter.flow(trainX, trainY, BATCHSIZE)\nvalidGenerator = validAugmenter.flow(validX, validY, BATCHSIZE)\ntestGenerator = testAugmenter.flow(testX, None, BATCHSIZE)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0e6b3524f4ac6e23eb03ca5b98c2d24a702c335"},"cell_type":"markdown","source":"The effect of data augmentation is hardly noticeable for is, but greatly improves the number of samples to train on. "},{"metadata":{"trusted":true,"_uuid":"625d61febfd759351bdf5f6495300e6687bf0958"},"cell_type":"code","source":"for batchX, batchY in trainGenerator:\n digits(batchX)\n break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2f9ceb1cd733ca9e28be52c813664ec8697ec2d"},"cell_type":"markdown","source":"### Model Architecture\n\nWe choose four convolutional layers and three dense layers."},{"metadata":{"trusted":true,"_uuid":"be657e40c7873d0153ea6459b5018b1409f06ebb"},"cell_type":"code","source":"NUMFILTERS = 16\nFILTER = 3\nACTIVATION = 'tanh'\nDROPOUT = 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b52b751cbe779fbfc3c2214a8bf86cff777a1bd8"},"cell_type":"code","source":"model = Sequential([\n Conv2D(NUMFILTERS, (FILTER, FILTER), padding='same', activation=ACTIVATION, input_shape=(trainX.shape[1:])),\n MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\n Conv2D(NUMFILTERS*2, (FILTER, FILTER), padding='same', activation=ACTIVATION),\n MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\n Conv2D(NUMFILTERS*4, (FILTER, FILTER), padding='same', activation=ACTIVATION),\n MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\n Conv2D(NUMFILTERS*8, (FILTER, FILTER), padding='same', activation=ACTIVATION),\n MaxPooling2D(pool_size=(2, 2), strides=(2, 2)),\n\n Flatten(),\n Dropout(DROPOUT),\n Dense(trainY.shape[1]*6, activation=ACTIVATION),\n Dropout(DROPOUT),\n Dense(trainY.shape[1]*2, activation=ACTIVATION),\n Dropout(DROPOUT),\n Dense(trainY.shape[1], activation='softmax'),\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94c970e245210b09f34998cf000ee1f1d2c8c6cb"},"cell_type":"markdown","source":"I noticed that it helps to manually set the learning rate lower than the default rate."},{"metadata":{"trusted":true,"_uuid":"17b82e1052e101895cbdfb1b63e0c6f2de5c63df"},"cell_type":"code","source":"model.compile(optimizer=optimizers.Adam(0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f3f5392ce5790fffcd3014c1411a5d11f8eb283"},"cell_type":"markdown","source":"### Model Training\nLet's train."},{"metadata":{"trusted":true,"_uuid":"9acb3c04bd9bbbdb3ca10821468676a38f0b6dea"},"cell_type":"code","source":"EPOCHS = 50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"847f90d189b5632164df67357bd637ae12d98ee8"},"cell_type":"code","source":"model.fit_generator(trainGenerator, epochs=EPOCHS, validation_data=validGenerator)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ffe474f78456b642538ce5bdf74dc1d283a456ae"},"cell_type":"markdown","source":"### Model Evaluation\nLet's plot the training evolution."},{"metadata":{"trusted":true,"_uuid":"e8e7c41d028b2961f6ea7420b4e8fe3fe86747d6"},"cell_type":"code","source":"x = np.arange(EPOCHS)+1\nhistory = model.history.history\n\ndata = [\n go.Scatter(x=x, y=history[\"acc\"], name=\"Train Accuracy\", marker=dict(size=5), yaxis='y2'),\n go.Scatter(x=x, y=history[\"val_acc\"], name=\"Valid Accuracy\", marker=dict(size=5), yaxis='y2'),\n go.Scatter(x=x, y=history[\"loss\"], name=\"Train Loss\", marker=dict(size=5)),\n go.Scatter(x=x, y=history[\"val_loss\"], name=\"Valid Loss\", marker=dict(size=5))\n]\nlayout = go.Layout(title=\"Model Training Evolution\", font=dict(family='Palatino'), \n xaxis=dict(title='Epoch', range=[0,EPOCHS]),\n yaxis1=dict(title=\"Loss\", domain=[0, 0.45]), \n yaxis2=dict(title=\"Accuracy\", domain=[0.55, 1]))\npy.iplot(go.Figure(data=data, layout=layout), show_link=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19f530d8401f8eec6c70e2a7905f5c937bf82be8"},"cell_type":"code","source":"print(model.evaluate(trainX, trainY))\nprint(model.evaluate(validX, validY))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b912581d718d178f8d098278cfe2e74d9cd658d9"},"cell_type":"markdown","source":"### Error Analysis\nError analysis gives us great insight in the way the model is making its errors. Often, it shows data quality issues."},{"metadata":{"trusted":true,"_uuid":"e7ed93102c055b6a39a4e57209d85f10c105605f"},"cell_type":"code","source":"validProbs = model.predict(validX)\nvalidPreds = validProbs.argmax(axis=1)\nvalidTruth = validY.argmax(axis=1)\nconfusionmatrix = pd.crosstab(validPreds, validTruth, normalize=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"846b46ac640616d0c0e93f327256409057d158a4"},"cell_type":"code","source":"trace = go.Heatmap(x=np.arange(10), y=np.arange(10), z=confusionmatrix.values,\n colorscale = [[0,\"#ffffff\"], [0.1,\"#000000\"], [1,\"#000000\"]],\n colorbar = dict(ticksuffix='', thickness=15, len=1))\nlayout = go.Layout(font=dict(family='Palatino'), height=500, width=600,\n margin = go.layout.Margin(l=50, r=30, b=30, t=50), \n xaxis=dict(title=\"Truth\", dtick=1, side='top'), \n yaxis=dict(title=\"Prediction\", dtick=1, autorange='reversed'))\npy.iplot(go.Figure(data=[trace], layout=layout), show_link=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aafd4a06778dcfa96a98eef787ef81c0727c852f"},"cell_type":"markdown","source":"Apparently, most errors come from predicting images of 9's as 4's. Some of the errors seem understandable."},{"metadata":{"trusted":true,"_uuid":"fcc5c6d66d2e0ad97be49d503cf8c26547258318"},"cell_type":"code","source":"digits(validX[(validTruth==4)&(validPreds==9)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0e5b797a38b4364a78736bb3e72835b0ed20035"},"cell_type":"markdown","source":"Here are images of 9's predicted as 8's."},{"metadata":{"trusted":true,"_uuid":"3a68fd2a963f4d12c07823167c97f1f9deff76b2"},"cell_type":"code","source":"digits(validX[(validTruth==9)&(validPreds==8)])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46c8ecf0077d1ae9371cbf4e8e64ce07806d4552"},"cell_type":"markdown","source":"We can also plot the most uncertain, and most certain predictions. Ones don't seem to be the problem."},{"metadata":{"trusted":true,"_uuid":"c6ed46318355052ee46b491e21c35895a07281f2"},"cell_type":"code","source":"order = (-validProbs.max(axis=1)).argsort()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b44cee58dd0070463491a8df99742849a5f438df"},"cell_type":"code","source":"plotCertainty(validX[order[-10:]], validPreds[order[-10:]], validProbs[order[-10:]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85f9ae90cf7e5d8d93941bbad3f6657526a9fcf9"},"cell_type":"code","source":"plotCertainty(validX[order[:10]], validPreds[order[:10]], validProbs[order[:10]])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4da776f191d08eb4682ca101212fe5d21c37640f"},"cell_type":"markdown","source":"### Test Predictions\nFinally, let's predict the test set."},{"metadata":{"trusted":true,"_uuid":"d4f02d97de577124645d04a753be7d16e3ad3224"},"cell_type":"code","source":"probabilities = model.predict(testX)\npredictions = probabilities.argmax(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5791efb25a459bf7fe0bb0a03ba6a8fc5202df8e"},"cell_type":"code","source":"submission = pd.DataFrame({\"ImageId\":np.arange(1, len(predictions)+1), \"Label\":predictions})\nsubmission.to_csv(\"submissionMnist.csv\", index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"hide_input":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":true,"title_cell":"Content","title_sidebar":"Contents","toc_cell":true,"toc_position":{},"toc_section_display":true,"toc_window_display":true},"varInspector":{"cols":{"lenName":16,"lenType":16,"lenVar":40},"kernels_config":{"python":{"delete_cmd_postfix":"","delete_cmd_prefix":"del ","library":"var_list.py","varRefreshCmd":"print(var_dic_list())"},"r":{"delete_cmd_postfix":") ","delete_cmd_prefix":"rm(","library":"var_list.r","varRefreshCmd":"cat(var_dic_list()) "}},"types_to_exclude":["module","function","builtin_function_or_method","instance","_Feature"],"window_display":false}},"nbformat":4,"nbformat_minor":1}