{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-14T10:25:24.892763Z","iopub.execute_input":"2021-12-14T10:25:24.893062Z","iopub.status.idle":"2021-12-14T10:25:25.117609Z","shell.execute_reply.started":"2021-12-14T10:25:24.893009Z","shell.execute_reply":"2021-12-14T10:25:25.116658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing a CNN on MNIST  \nHow does a Convolutional Neural Network (CNN) perfom on the MNIST Digits data set?\n\nref: [Digit Recogniser Competition](https://www.kaggle.com/c/digit-recognizer)\n \nscores ~0.992","metadata":{}},{"cell_type":"markdown","source":"### Imports required","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport matplotlib\nimport matplotlib.pyplot as plt\nfrom keras.layers import Dense\nfrom keras.layers import Dropout\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten\nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nfrom keras.callbacks import ReduceLROnPlateau, EarlyStopping\nfrom sklearn.metrics import accuracy_score\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:25:31.316814Z","iopub.execute_input":"2021-12-14T10:25:31.317123Z","iopub.status.idle":"2021-12-14T10:25:33.584381Z","shell.execute_reply.started":"2021-12-14T10:25:31.317074Z","shell.execute_reply":"2021-12-14T10:25:33.583611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Have a look at the data and prepare it.","metadata":{}},{"cell_type":"code","source":"# load the training data\nX = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX.shape","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-12-14T10:25:33.586076Z","iopub.execute_input":"2021-12-14T10:25:33.586395Z","iopub.status.idle":"2021-12-14T10:25:36.747145Z","shell.execute_reply.started":"2021-12-14T10:25:33.586337Z","shell.execute_reply":"2021-12-14T10:25:36.746451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:25:37.817258Z","iopub.execute_input":"2021-12-14T10:25:37.817612Z","iopub.status.idle":"2021-12-14T10:25:37.845896Z","shell.execute_reply.started":"2021-12-14T10:25:37.817554Z","shell.execute_reply":"2021-12-14T10:25:37.845244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# what is the highest value?\nX.max().max()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:26:07.521719Z","iopub.execute_input":"2021-12-14T10:26:07.525274Z","iopub.status.idle":"2021-12-14T10:26:07.599346Z","shell.execute_reply.started":"2021-12-14T10:26:07.524906Z","shell.execute_reply":"2021-12-14T10:26:07.598445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare the training Data","metadata":{}},{"cell_type":"code","source":"y = X['label']\nX.drop(['label'], axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:27:33.490761Z","iopub.execute_input":"2021-12-14T10:27:33.491079Z","iopub.status.idle":"2021-12-14T10:27:33.661377Z","shell.execute_reply.started":"2021-12-14T10:27:33.491027Z","shell.execute_reply":"2021-12-14T10:27:33.659924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### What do the original images look like?","metadata":{}},{"cell_type":"code","source":"# change the range to see more!\nfor i in range(3):\n    # convert to an array and reshape\n    a_digit = X.loc[i].to_numpy().reshape(28, 28) \n    plt.imshow(a_digit, cmap=matplotlib.cm.binary)\n    print('The digit label', y.loc[i])\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:33:15.303233Z","iopub.execute_input":"2021-12-14T10:33:15.303573Z","iopub.status.idle":"2021-12-14T10:33:15.685802Z","shell.execute_reply.started":"2021-12-14T10:33:15.303521Z","shell.execute_reply":"2021-12-14T10:33:15.685151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Scale the data  \nScaling the data will improve the convergence speed and the accuracy of the model.","metadata":{"execution":{"iopub.status.busy":"2021-12-14T09:51:01.816173Z","iopub.execute_input":"2021-12-14T09:51:01.816793Z","iopub.status.idle":"2021-12-14T09:51:01.985794Z","shell.execute_reply.started":"2021-12-14T09:51:01.816741Z","shell.execute_reply":"2021-12-14T09:51:01.985079Z"}}},{"cell_type":"code","source":"# normalise the data by dividing by the max (as a float)\nX /= 255.0\n# convert X to an array\nX = np.array(X)","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:46:48.117179Z","iopub.execute_input":"2021-12-14T10:46:48.117533Z","iopub.status.idle":"2021-12-14T10:46:48.999294Z","shell.execute_reply.started":"2021-12-14T10:46:48.117457Z","shell.execute_reply":"2021-12-14T10:46:48.998532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reshape to images  \nA CNN requires that the the flattened array provided is reshaped back into layers of squares , so the filter can be passed over the image correctly","metadata":{}},{"cell_type":"code","source":"# reshape to be [samples][width][height][channels]\nXr = X.reshape(X.shape[0], 28, 28, 1)\n# check the shape\nXr.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:46:50.820119Z","iopub.execute_input":"2021-12-14T10:46:50.820648Z","iopub.status.idle":"2021-12-14T10:46:50.829023Z","shell.execute_reply.started":"2021-12-14T10:46:50.820448Z","shell.execute_reply":"2021-12-14T10:46:50.827921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert class vectors to binary class matrices\nnum_classes = 10\ny = to_categorical(y, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-12-14T10:46:51.770615Z","iopub.execute_input":"2021-12-14T10:46:51.770901Z","iopub.status.idle":"2021-12-14T10:46:51.775331Z","shell.execute_reply.started":"2021-12-14T10:46:51.770852Z","shell.execute_reply":"2021-12-14T10:46:51.774571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# split into a training and validation set\nX_train, X_val, y_train, y_val = train_test_split(Xr, y, test_size=0.2, random_state=42, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:11:09.269433Z","iopub.execute_input":"2021-12-14T11:11:09.269763Z","iopub.status.idle":"2021-12-14T11:11:09.880842Z","shell.execute_reply.started":"2021-12-14T11:11:09.269709Z","shell.execute_reply":"2021-12-14T11:11:09.880101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Set up the CNN","metadata":{}},{"cell_type":"code","source":"# callback to reduce the learning rate \nred_lr = ReduceLROnPlateau(monitor='val_acc', patience=2, verbose=1, mode='max' )\n# stop the epochs when there is no more improvement in the validation set\nes = EarlyStopping(monitor='val_acc', mode='max', verbose=1, patience=5)\n\nmodel = Sequential()\n\nmodel.add(Conv2D(64, input_shape=(28, 28, 1) , kernel_size=(7,7), activation='elu'))\nmodel.add(MaxPooling2D(pool_size=(3, 3)))\nmodel.add(Conv2D(64, kernel_size=(3,3), activation='elu'))\nmodel.add(Conv2D(64, kernel_size=(3,3), activation='elu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Flatten())\nmodel.add(Dense(300, activation='elu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(300, activation='elu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))\n\n# Compile the model\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nprint(model.summary())\n\n#Train the model\nmodel_his = model.fit(X_train, y_train, \n                      batch_size=50,\n                      epochs=50, \n                      callbacks=[es, red_lr],\n                      validation_data=(X_val, y_val) \n                      )\n","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:19:24.231485Z","iopub.execute_input":"2021-12-14T11:19:24.231853Z","iopub.status.idle":"2021-12-14T11:20:34.752396Z","shell.execute_reply.started":"2021-12-14T11:19:24.231795Z","shell.execute_reply":"2021-12-14T11:20:34.75129Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Monitor the training","metadata":{}},{"cell_type":"code","source":"# Plot train vs test accuracy per epoch\nplt.figure()\n# Use the history metrics\nplt.plot(model_his.history['acc'])\nplt.plot(model_his.history['val_acc'])\n# Make it pretty\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train', 'Test'])\n# the highest accuracy score for the adding another hidden layer\nprint('\\nThe highest test score', max(model_his.history['val_acc']).round(3))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:21:08.948107Z","iopub.execute_input":"2021-12-14T11:21:08.948448Z","iopub.status.idle":"2021-12-14T11:21:09.104385Z","shell.execute_reply.started":"2021-12-14T11:21:08.948389Z","shell.execute_reply":"2021-12-14T11:21:09.103418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Which images is the model predicting incorrectly?","metadata":{}},{"cell_type":"code","source":"val_preds = model.predict(X_val)\nval_numbers = np.argmax(val_preds, axis=1)\ny_val_numbers = np.argmax(y_val, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:24:25.695669Z","iopub.execute_input":"2021-12-14T11:24:25.696213Z","iopub.status.idle":"2021-12-14T11:24:26.799162Z","shell.execute_reply.started":"2021-12-14T11:24:25.695992Z","shell.execute_reply":"2021-12-14T11:24:26.798435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a df to hold the results\nresult = pd.DataFrame({'predicted label' : val_numbers, 'actual labels' : y_val_numbers})\n# label the incorrect results\nresult['correct'] = result['predicted label'] == result['actual labels']\n# select the incorrect results\nwrong = result[result['correct'] == False]\n\n# which numbers did the model get wrong?\nwrong['actual labels'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:26:39.648292Z","iopub.execute_input":"2021-12-14T11:26:39.648639Z","iopub.status.idle":"2021-12-14T11:26:39.663294Z","shell.execute_reply.started":"2021-12-14T11:26:39.648586Z","shell.execute_reply":"2021-12-14T11:26:39.66209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There isn't a single number the model gets wrong, it performs better on 0's ,1's and 6's.\nLets have a look at some of the misidentified numbers.","metadata":{}},{"cell_type":"code","source":"w_list = wrong[:18].index\nplt.figure(figsize=(9, 12))\nfor i, v in enumerate(w_list):\n    plt.subplot(6,3,i+1)\n    plt.imshow(X_val[v][:,:,0], cmap=matplotlib.cm.binary)\n    plt.title('Pred: {} Act: {}'.format(wrong['predicted label'][v], wrong['actual labels'][v]))\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-12-14T11:30:59.513161Z","iopub.execute_input":"2021-12-14T11:30:59.513504Z","iopub.status.idle":"2021-12-14T11:31:01.672186Z","shell.execute_reply.started":"2021-12-14T11:30:59.513447Z","shell.execute_reply":"2021-12-14T11:31:01.67092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the majority of these its clear to see why the model got them wrong and I already think it may be overfitting, possibly to the validation set. I do not think it is worth trying to improve the model any further for now. The nest step is to try it on the withheld test set and see how the model performs on unseen data.","metadata":{}},{"cell_type":"markdown","source":"## Predict the test set  \n### Prepare the test data","metadata":{}},{"cell_type":"code","source":"# load the test data\nX_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n# check the shape\nX_test.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# normalise the data by dividing by the max (as a float)\nX_test /= 255.0\n# convert X_test to an array\nX_test = np.array(X_test)\n# reshape to be [samples][width][height][channels]\nX_test = X_test.reshape(X_test.shape[0], 28, 28, 1)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict the test set\npreds = model.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What does the first predcition look like?\npreds[0]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert back from a categorical array to an array of the digits predicted\nnum_preds = np.argmax(preds, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the submission template\nsub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replace the label values with the model predictions\nsub['Label'] = num_preds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check that it worked\nsub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# output the submission file\nsub.to_csv('./submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}