{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Import the necessary libs\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.datasets import mnist\nimport tensorflow as tf\n\nsns.set(style='white', context='notebook', palette='deep')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-24T19:13:59.305323Z","iopub.execute_input":"2021-10-24T19:13:59.305711Z","iopub.status.idle":"2021-10-24T19:14:06.353682Z","shell.execute_reply.started":"2021-10-24T19:13:59.305678Z","shell.execute_reply":"2021-10-24T19:14:06.352674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\nsub = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:21.195320Z","iopub.execute_input":"2021-10-24T19:14:21.195715Z","iopub.status.idle":"2021-10-24T19:14:26.009741Z","shell.execute_reply.started":"2021-10-24T19:14:21.195679Z","shell.execute_reply":"2021-10-24T19:14:26.008842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training data size is {train.shape}\\nTesting data size is {test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:30.121347Z","iopub.execute_input":"2021-10-24T19:14:30.121895Z","iopub.status.idle":"2021-10-24T19:14:30.128193Z","shell.execute_reply.started":"2021-10-24T19:14:30.121860Z","shell.execute_reply":"2021-10-24T19:14:30.126644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:32.679100Z","iopub.execute_input":"2021-10-24T19:14:32.679619Z","iopub.status.idle":"2021-10-24T19:14:32.714464Z","shell.execute_reply.started":"2021-10-24T19:14:32.679588Z","shell.execute_reply":"2021-10-24T19:14:32.713729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-24T13:44:44.425631Z","iopub.execute_input":"2021-10-24T13:44:44.426030Z","iopub.status.idle":"2021-10-24T13:44:44.448089Z","shell.execute_reply.started":"2021-10-24T13:44:44.425995Z","shell.execute_reply":"2021-10-24T13:44:44.446729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"], axis = 1) ","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:37.122808Z","iopub.execute_input":"2021-10-24T19:14:37.123398Z","iopub.status.idle":"2021-10-24T19:14:37.215301Z","shell.execute_reply.started":"2021-10-24T19:14:37.123351Z","shell.execute_reply":"2021-10-24T19:14:37.214113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print data histogram\nsns.countplot(Y_train);","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:40.286378Z","iopub.execute_input":"2021-10-24T19:14:40.286782Z","iopub.status.idle":"2021-10-24T19:14:40.696947Z","shell.execute_reply.started":"2021-10-24T19:14:40.286746Z","shell.execute_reply":"2021-10-24T19:14:40.695784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We perform a grayscale normalization to reduce the effect of illumination's differences. \n\nMoreover the CNN converg faster on [0..1] data than on [0..255].","metadata":{}},{"cell_type":"code","source":"# Normalize data to make CNN faster\nX_train = X_train / 255.0\n\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:53.082697Z","iopub.execute_input":"2021-10-24T19:14:53.083162Z","iopub.status.idle":"2021-10-24T19:14:53.217601Z","shell.execute_reply.started":"2021-10-24T19:14:53.083119Z","shell.execute_reply":"2021-10-24T19:14:53.216717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\n# canal = 1 => For gray scale\nX_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:55.646972Z","iopub.execute_input":"2021-10-24T19:14:55.647738Z","iopub.status.idle":"2021-10-24T19:14:55.654062Z","shell.execute_reply.started":"2021-10-24T19:14:55.647687Z","shell.execute_reply":"2021-10-24T19:14:55.652870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**One-Hot Encoding**","metadata":{}},{"cell_type":"code","source":"# Convert label to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:14:59.188404Z","iopub.execute_input":"2021-10-24T19:14:59.189031Z","iopub.status.idle":"2021-10-24T19:14:59.193417Z","shell.execute_reply.started":"2021-10-24T19:14:59.188995Z","shell.execute_reply":"2021-10-24T19:14:59.192600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Labels are 10 digits numbers from 0 to 9. We need to encode these lables to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0]).","metadata":{}},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_tr, X_val, Y_tr, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:15:02.601933Z","iopub.execute_input":"2021-10-24T19:15:02.602543Z","iopub.status.idle":"2021-10-24T19:15:03.017360Z","shell.execute_reply.started":"2021-10-24T19:15:02.602510Z","shell.execute_reply":"2021-10-24T19:15:03.016424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tr.shape, X_val.shape, Y_tr.shape, Y_val.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:15:52.271592Z","iopub.execute_input":"2021-10-24T19:15:52.272264Z","iopub.status.idle":"2021-10-24T19:15:52.279272Z","shell.execute_reply.started":"2021-10-24T19:15:52.272218Z","shell.execute_reply":"2021-10-24T19:15:52.278130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Data Visualization**","metadata":{}},{"cell_type":"code","source":"# Draw an example of a data set to see\ng = plt.imshow(X_tr[189][:,:,0])","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:15:55.335386Z","iopub.execute_input":"2021-10-24T19:15:55.335783Z","iopub.status.idle":"2021-10-24T19:15:55.543956Z","shell.execute_reply.started":"2021-10-24T19:15:55.335749Z","shell.execute_reply":"2021-10-24T19:15:55.542883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Definition**","metadata":{}},{"cell_type":"code","source":"#Creating CNN model\n\"\"\"\n  [[Conv2D->relu]*2 -> BatchNormalization -> MaxPool2D -> Dropout]*2 -> \n  [Conv2D->relu]*2 -> BatchNormalization -> Dropout -> \n  Flatten -> Dense -> BatchNormalization -> Dropout -> Out\n\"\"\"\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 64, kernel_size = (5,5),padding = 'Same', activation ='relu', input_shape = (28,28,1)))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 64, kernel_size = (5,5),padding = 'Same', activation ='relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\nmodel.add(BatchNormalization())\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', activation ='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3), padding = 'Same',  activation ='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:15:59.354097Z","iopub.execute_input":"2021-10-24T19:15:59.354569Z","iopub.status.idle":"2021-10-24T19:15:59.648091Z","shell.execute_reply.started":"2021-10-24T19:15:59.354532Z","shell.execute_reply":"2021-10-24T19:15:59.646772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Plot CNN model**","metadata":{}},{"cell_type":"code","source":"# print out model look\nfrom keras.utils import plot_model\nplot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)\nfrom IPython.display import Image\nImage(\"model.png\")","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:16:03.005258Z","iopub.execute_input":"2021-10-24T19:16:03.005631Z","iopub.status.idle":"2021-10-24T19:16:03.671083Z","shell.execute_reply.started":"2021-10-24T19:16:03.005598Z","shell.execute_reply":"2021-10-24T19:16:03.669817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = Adam(), loss = 'categorical_crossentropy', metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:16:08.902294Z","iopub.execute_input":"2021-10-24T19:16:08.902689Z","iopub.status.idle":"2021-10-24T19:16:08.926522Z","shell.execute_reply.started":"2021-10-24T19:16:08.902628Z","shell.execute_reply":"2021-10-24T19:16:08.925684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Audjusting learning rate\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:16:12.773385Z","iopub.execute_input":"2021-10-24T19:16:12.774115Z","iopub.status.idle":"2021-10-24T19:16:12.779182Z","shell.execute_reply.started":"2021-10-24T19:16:12.774074Z","shell.execute_reply":"2021-10-24T19:16:12.778142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Adjusting epochs and batch_size\nepochs = 50\nbatch_size = 128","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:16:15.476005Z","iopub.execute_input":"2021-10-24T19:16:15.476389Z","iopub.status.idle":"2021-10-24T19:16:15.480953Z","shell.execute_reply.started":"2021-10-24T19:16:15.476358Z","shell.execute_reply":"2021-10-24T19:16:15.479905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" Data augmentation","metadata":{}},{"cell_type":"code","source":"#Data Augmentation \ndatagen = ImageDataGenerator(\n        featurewise_center=False, # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n#datagen.fit(X_train)\ntrain_gen = datagen.flow(X_tr,Y_tr, batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:16:23.884832Z","iopub.execute_input":"2021-10-24T19:16:23.885626Z","iopub.status.idle":"2021-10-24T19:16:23.963284Z","shell.execute_reply.started":"2021-10-24T19:16:23.885571Z","shell.execute_reply":"2021-10-24T19:16:23.962340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For the data augmentation, i choosed to :\n   - Randomly rotate some training images by 10 degrees\n   - Randomly  Zoom by 10% some training images\n   - Randomly shift images horizontally by 10% of the width\n   - Randomly shift images vertically by 10% of the height\n   \nI did not apply a vertical_flip nor horizontal_flip since it could have lead to misclassify symetrical numbers such as 6 and 9.\n\nOnce our model is ready, we fit the training dataset .","metadata":{}},{"cell_type":"markdown","source":"Model training","metadata":{}},{"cell_type":"code","source":"#Prediction model\nhistory = model.fit(train_gen,\n                    epochs = epochs,validation_data = (X_val,Y_val),verbose = 2, \n                    callbacks=[learning_rate_reduction])","metadata":{"execution":{"iopub.status.busy":"2021-10-24T13:52:24.225273Z","iopub.execute_input":"2021-10-24T13:52:24.225680Z","iopub.status.idle":"2021-10-24T19:03:45.319487Z","shell.execute_reply.started":"2021-10-24T13:52:24.225646Z","shell.execute_reply":"2021-10-24T19:03:45.317485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make predictions about test sets\nresults = model.predict(test)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:21:52.760204Z","iopub.execute_input":"2021-10-24T19:21:52.760601Z","iopub.status.idle":"2021-10-24T19:22:51.076055Z","shell.execute_reply.started":"2021-10-24T19:21:52.760571Z","shell.execute_reply":"2021-10-24T19:22:51.074845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = []\nfor i in range(test.shape[0]): labels.append(np.argmax(results[i]))\nsubmission = pd.DataFrame({'ImageID': [i+1 for i in range(test.shape[0])], 'Label': labels,})\nsubmission.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:25:02.598726Z","iopub.execute_input":"2021-10-24T19:25:02.599276Z","iopub.status.idle":"2021-10-24T19:25:02.742409Z","shell.execute_reply.started":"2021-10-24T19:25:02.599244Z","shell.execute_reply":"2021-10-24T19:25:02.741258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-24T19:27:51.150474Z","iopub.execute_input":"2021-10-24T19:27:51.150865Z","iopub.status.idle":"2021-10-24T19:27:51.212513Z","shell.execute_reply.started":"2021-10-24T19:27:51.150835Z","shell.execute_reply":"2021-10-24T19:27:51.211450Z"},"trusted":true},"execution_count":null,"outputs":[]}]}