{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Introduction\n\n\n\n\n\n<img src=\"http://img1.imagilive.com/0717/mnist-sample.png\" ></img>","metadata":{"_uuid":"e9aff3cf1bb8daa73bec67b970d12195677679f3","_cell_guid":"eb88b372-a6e5-40c8-a1c6-c03799165490"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nsns.set(style='white', context='notebook', palette='deep')","metadata":{"_execution_state":"idle","_uuid":"72334cb006d02a4bcfc2a2fe622524eba824c6f8","_cell_guid":"f67b9393-8ea1-4e23-b856-2ce149cfe421","execution":{"iopub.status.busy":"2023-10-18T05:06:14.539991Z","iopub.execute_input":"2023-10-18T05:06:14.540576Z","iopub.status.idle":"2023-10-18T05:06:24.851860Z","shell.execute_reply.started":"2023-10-18T05:06:14.540538Z","shell.execute_reply":"2023-10-18T05:06:24.850679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Data preparation\n## 2.1 Load data","metadata":{"_execution_state":"idle","_uuid":"86061d98eccaa02efe0dab0fa3884e71fcf4c310","_cell_guid":"6d2fb3e6-ab71-4974-b5a2-4af1ebdb99f4"}},{"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","metadata":{"_execution_state":"idle","_uuid":"84bbd5ab8d7895bd430d5ecfe2f7ddf77baa7b74","_cell_guid":"5e51d00e-62fd-4141-bf73-50ac4f2da7d0","execution":{"iopub.status.busy":"2023-10-18T05:06:24.853445Z","iopub.execute_input":"2023-10-18T05:06:24.854140Z","iopub.status.idle":"2023-10-18T05:06:30.210684Z","shell.execute_reply.started":"2023-10-18T05:06:24.854104Z","shell.execute_reply":"2023-10-18T05:06:30.209882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = train[\"label\"]\n\n# Drop 'label' column\nX_train = train.drop(labels = [\"label\"],axis = 1) \n\n# free some space\ndel train \n\ng = sns.countplot(Y_train)\n\nY_train.value_counts()","metadata":{"_execution_state":"idle","_uuid":"1213b979d5ed3e0d13824d17d694c79d2ece92fa","_cell_guid":"86570a36-5c20-460a-9dfd-2070548532a7","execution":{"iopub.status.busy":"2023-10-18T05:06:30.212084Z","iopub.execute_input":"2023-10-18T05:06:30.212485Z","iopub.status.idle":"2023-10-18T05:06:30.584170Z","shell.execute_reply.started":"2023-10-18T05:06:30.212447Z","shell.execute_reply":"2023-10-18T05:06:30.583170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have similar counts for the 10 digits.","metadata":{"_uuid":"b7b1b1d36243c885e57374c8b60c5a7e10abe922","_cell_guid":"5aea4062-1790-4987-b739-c4bebd79030f"}},{"cell_type":"markdown","source":"## 2.2 Check for null and missing values","metadata":{"_uuid":"5d77934302869925c19128c77e247b3c8ca84d71","_cell_guid":"5b7d4b66-a140-4fcc-a889-bcef007c880a"}},{"cell_type":"code","source":"# Check the data\nX_train.isnull().any().describe()","metadata":{"_execution_state":"idle","_uuid":"cdf27c27e2a5b15e6d7bfc70de7a18c08f3feb7a","_cell_guid":"ececaa00-2ae3-4d13-b631-438df085b030","execution":{"iopub.status.busy":"2023-10-18T05:06:30.586201Z","iopub.execute_input":"2023-10-18T05:06:30.586935Z","iopub.status.idle":"2023-10-18T05:06:30.614960Z","shell.execute_reply.started":"2023-10-18T05:06:30.586901Z","shell.execute_reply":"2023-10-18T05:06:30.613874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().describe()","metadata":{"_execution_state":"idle","_uuid":"a0089bb7ec9aec76373db475399aea24699ae989","_cell_guid":"92572e80-8543-4e72-8767-5c9be8381b04","execution":{"iopub.status.busy":"2023-10-18T05:06:30.616369Z","iopub.execute_input":"2023-10-18T05:06:30.616689Z","iopub.status.idle":"2023-10-18T05:06:30.632642Z","shell.execute_reply.started":"2023-10-18T05:06:30.616661Z","shell.execute_reply":"2023-10-18T05:06:30.631798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I check for corrupted images (missing values inside).\n\nThere is no missing values in the train and test dataset. So we can safely go ahead.","metadata":{"_uuid":"c0bee59691c2df0b275c78e38e7f9907d02ac038","_cell_guid":"09d04cae-4245-4659-85dd-ef48531da295"}},{"cell_type":"markdown","source":"## 2.3 Normalization","metadata":{"_uuid":"2954681f25f0dcbe986e6914396cdbce61db591f","_cell_guid":"6812040d-80ad-43d2-a571-275f4f20067b"}},{"cell_type":"markdown","source":"We perform a grayscale normalization to reduce the effect of illumination's differences. \n\nMoreover the CNN converg faster on [0..1] data than on [0..255].","metadata":{"_uuid":"0ecf4b52510ab7957d0d4eb646c0aa1ba5986273","_cell_guid":"159d5854-437a-4d0f-bc1e-fc3f7e43d178"}},{"cell_type":"code","source":"# Normalize the data\nX_train = X_train / 255.0\ntest = test / 255.0","metadata":{"_execution_state":"idle","_uuid":"b5d4f8fcf2a967e2c7d57daedf95aa8c5ab7f8cb","_cell_guid":"cdc4340b-6e24-4e12-be99-ac806098ff17","execution":{"iopub.status.busy":"2023-10-18T05:06:30.633950Z","iopub.execute_input":"2023-10-18T05:06:30.634337Z","iopub.status.idle":"2023-10-18T05:06:30.746739Z","shell.execute_reply.started":"2023-10-18T05:06:30.634298Z","shell.execute_reply":"2023-10-18T05:06:30.745873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2.3 Reshape","metadata":{"_uuid":"a66741bf1ac597094f3a3166877008feef27c519","_cell_guid":"7413df94-bcb9-4f75-b174-c127d4445766"}},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nX_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","metadata":{"_execution_state":"idle","_uuid":"f0a6ad80dab8e0f2c2e46165ccd9cd82dd162bc3","_cell_guid":"34b6a5f7-8fd2-4387-8ef4-c9dc19584fed","execution":{"iopub.status.busy":"2023-10-18T05:06:30.747597Z","iopub.execute_input":"2023-10-18T05:06:30.747957Z","iopub.status.idle":"2023-10-18T05:06:30.753477Z","shell.execute_reply.started":"2023-10-18T05:06:30.747929Z","shell.execute_reply":"2023-10-18T05:06:30.752365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train and test images (28px x 28px) has been stock into pandas.Dataframe as 1D vectors of 784 values. We reshape all data to 28x28x1 3D matrices. \n\nKeras requires an extra dimension in the end which correspond to channels. MNIST images are gray scaled so it use only one channel. For RGB images, there is 3 channels, we would have reshaped 784px vectors to 28x28x3 3D matrices. ","metadata":{"_uuid":"f4fb5553e188d9956f5d8b3a5d275ab00ea667ce","_cell_guid":"8decd1ce-7b7e-431d-8458-eaca18e0e1f7"}},{"cell_type":"markdown","source":"## 2.5 Label encoding","metadata":{"_uuid":"39b7a31e843bac6b705461bcce89da216b91799e","_cell_guid":"bdb422e2-bdec-444f-97a5-283a1e54bf2c"}},{"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"_execution_state":"idle","_uuid":"cabefd1478d5c1bdfe57fd6a34395340916a854c","_cell_guid":"4b7f3e78-44dc-4561-b1f0-9429ee024cf4","execution":{"iopub.status.busy":"2023-10-18T05:06:30.754947Z","iopub.execute_input":"2023-10-18T05:06:30.755598Z","iopub.status.idle":"2023-10-18T05:06:30.767231Z","shell.execute_reply.started":"2023-10-18T05:06:30.755565Z","shell.execute_reply":"2023-10-18T05:06:30.766244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Labels are 10 digits numbers from 0 to 9. We need to encode these lables to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0]).","metadata":{"_uuid":"dcfb688587dfc6feafd27442a3505e35dc01b82d","_cell_guid":"ae068bd8-b12e-4768-8a7e-0fc865dd7562"}},{"cell_type":"markdown","source":"## 2.6 Split training and valdiation set ","metadata":{"_uuid":"d8abbbf31483b94e1b29d07c4c8253d1311648a7","_cell_guid":"32152fc3-a570-4d64-8a7d-6c689a4acd33"}},{"cell_type":"code","source":"# Set the random seed\nrandom_seed = 2","metadata":{"_execution_state":"idle","_uuid":"6e51c925c6e0f1b936679c9649fef345c853555f","_cell_guid":"3a698301-9759-4279-ae48-fd980f89ea53","execution":{"iopub.status.busy":"2023-10-18T05:06:30.768729Z","iopub.execute_input":"2023-10-18T05:06:30.769152Z","iopub.status.idle":"2023-10-18T05:06:30.778515Z","shell.execute_reply.started":"2023-10-18T05:06:30.769112Z","shell.execute_reply":"2023-10-18T05:06:30.777731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)","metadata":{"_execution_state":"idle","_uuid":"b779ac76d8317647db92d5a88b4098d212d72884","_cell_guid":"dcd25ebb-d845-4d32-9867-082e352b1396","execution":{"iopub.status.busy":"2023-10-18T05:06:30.781696Z","iopub.execute_input":"2023-10-18T05:06:30.782436Z","iopub.status.idle":"2023-10-18T05:06:31.204742Z","shell.execute_reply.started":"2023-10-18T05:06:30.782405Z","shell.execute_reply":"2023-10-18T05:06:31.203406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I choosed to split the train set in two parts : a small fraction (10%) became the validation set which the model is evaluated and the rest (90%) is used to train the model.\n\nSince we have 42 000 training images of balanced labels (see 2.1 Load data), a random split of the train set doesn't cause some labels to be over represented in the validation set. Be carefull with some unbalanced dataset a simple random split could cause inaccurate evaluation during the validation. \n\nTo avoid that, you could use stratify = True option in train_test_split function (**Only for >=0.17 sklearn versions**).","metadata":{"_uuid":"72ed54a305eebf80e8b8b0f7eabde1332a5a85a3","_cell_guid":"3c99964f-4500-4f1c-947d-c67e644e34db"}},{"cell_type":"markdown","source":"We can get a better sense for one of these examples by visualising the image and looking at the label.","metadata":{"_uuid":"60eed15ec5bc0d354385301789ecb8538fc02267","_cell_guid":"adbeacf0-0dc0-4675-b2df-9c9663750f32"}},{"cell_type":"code","source":"# Some examples\ng = plt.imshow(X_train[0][:,:,0])","metadata":{"_execution_state":"idle","_uuid":"e0dae8943d3d35f075dba3d7ba31bde1d4bf2ff4","_cell_guid":"5f76131b-4ba0-45f1-a98c-bd4e7d561793","execution":{"iopub.status.busy":"2023-10-18T05:06:31.206398Z","iopub.execute_input":"2023-10-18T05:06:31.207080Z","iopub.status.idle":"2023-10-18T05:06:31.531032Z","shell.execute_reply.started":"2023-10-18T05:06:31.207038Z","shell.execute_reply":"2023-10-18T05:06:31.529547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. CNN\n## 3.1 Define the model -- change this!","metadata":{"_uuid":"5fa18b37a9acd9e098bac1d12264b0dd4310fdd3","_cell_guid":"d5265777-aeb3-449d-b171-d88cad74c0a4"}},{"cell_type":"code","source":"# Set the CNN model \nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"_execution_state":"idle","_uuid":"f7991ef6871a26f9fa57acdcd460a69bab53e804","_cell_guid":"1e0f3f88-2ad7-459e-8e02-aecc5f3511ae","execution":{"iopub.status.busy":"2023-10-18T05:06:31.532755Z","iopub.execute_input":"2023-10-18T05:06:31.533442Z","iopub.status.idle":"2023-10-18T05:06:31.797647Z","shell.execute_reply.started":"2023-10-18T05:06:31.533405Z","shell.execute_reply":"2023-10-18T05:06:31.796781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the optimizer\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","metadata":{"_execution_state":"idle","_uuid":"420c704367b397b8255fefe9d882b35ac8929b95","_cell_guid":"a4c55409-6a65-400a-b5e8-a1dc535429c0","execution":{"iopub.status.busy":"2023-10-18T05:06:31.800669Z","iopub.execute_input":"2023-10-18T05:06:31.801107Z","iopub.status.idle":"2023-10-18T05:06:31.808255Z","shell.execute_reply.started":"2023-10-18T05:06:31.801061Z","shell.execute_reply":"2023-10-18T05:06:31.807028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"_execution_state":"idle","_uuid":"b4c0288622227a3cd05479aa765e324dbb852f34","_cell_guid":"0bd89b79-1a74-40da-af9a-7a76a8b96ff2","execution":{"iopub.status.busy":"2023-10-18T05:06:31.809996Z","iopub.execute_input":"2023-10-18T05:06:31.810407Z","iopub.status.idle":"2023-10-18T05:06:31.833516Z","shell.execute_reply.started":"2023-10-18T05:06:31.810371Z","shell.execute_reply":"2023-10-18T05:06:31.832320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<img src=\"http://img1.imagilive.com/0717/learningrates.jpg\"> </img>","metadata":{"_uuid":"f4d89b29d55b328e8c446880505c54e674eeec38","_cell_guid":"6bc49be3-11e5-48fb-b70f-601aec0f0c86"}},{"cell_type":"code","source":"# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","metadata":{"_execution_state":"idle","_uuid":"c4a5b4e462ec5362c47eef4fcc7956fd4e203307","_cell_guid":"b5987a18-6bbe-42a2-9d31-333ebc4f7af1","execution":{"iopub.status.busy":"2023-10-18T05:06:31.835253Z","iopub.execute_input":"2023-10-18T05:06:31.835641Z","iopub.status.idle":"2023-10-18T05:06:31.841542Z","shell.execute_reply.started":"2023-10-18T05:06:31.835607Z","shell.execute_reply":"2023-10-18T05:06:31.840225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 1 \nbatch_size = 86","metadata":{"_execution_state":"idle","_uuid":"26b0647c46efdb6b1096cf7335a7bf2a3417543a","_cell_guid":"970db455-b393-4b25-806d-92c6766c12c0","execution":{"iopub.status.busy":"2023-10-18T05:06:31.843120Z","iopub.execute_input":"2023-10-18T05:06:31.843564Z","iopub.status.idle":"2023-10-18T05:06:31.853321Z","shell.execute_reply.started":"2023-10-18T05:06:31.843529Z","shell.execute_reply":"2023-10-18T05:06:31.852356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\ndatagen.fit(X_train)","metadata":{"_execution_state":"idle","_uuid":"21d6192c87d92d497c797656474bccd9cefc5647","_cell_guid":"b342befe-1a6f-44bf-8dab-28033a729122","execution":{"iopub.status.busy":"2023-10-18T05:06:31.865970Z","iopub.execute_input":"2023-10-18T05:06:31.866341Z","iopub.status.idle":"2023-10-18T05:06:31.971209Z","shell.execute_reply.started":"2023-10-18T05:06:31.866309Z","shell.execute_reply":"2023-10-18T05:06:31.970162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size\n                              , callbacks=[learning_rate_reduction])","metadata":{"_execution_state":"idle","_uuid":"cf36b3d029f95b553be02d612e097a9769ee8252","_cell_guid":"b453af8d-9736-43e3-b486-7a1cd7dd8909","execution":{"iopub.status.busy":"2023-10-18T05:06:31.972459Z","iopub.execute_input":"2023-10-18T05:06:31.973359Z","iopub.status.idle":"2023-10-18T05:08:50.320068Z","shell.execute_reply.started":"2023-10-18T05:06:31.973328Z","shell.execute_reply":"2023-10-18T05:08:50.319166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"_execution_state":"idle","_uuid":"7f17e7bf0a54a01a52fef2d554780f6bc6580dc6","_cell_guid":"05ff3b9f-c3bb-4cec-a8c2-2c128e8f15b3","execution":{"iopub.status.busy":"2023-10-18T05:09:46.123648Z","iopub.execute_input":"2023-10-18T05:09:46.124794Z","iopub.status.idle":"2023-10-18T05:10:08.738394Z","shell.execute_reply.started":"2023-10-18T05:09:46.124757Z","shell.execute_reply":"2023-10-18T05:10:08.737266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","metadata":{"_execution_state":"idle","_uuid":"369dfaab09240f3f12bcff91953ffd315ab84985","_cell_guid":"b5f1f39f-13b8-439a-8913-0f120e3d47a9","execution":{"iopub.status.busy":"2023-10-18T05:10:36.992504Z","iopub.execute_input":"2023-10-18T05:10:36.992923Z","iopub.status.idle":"2023-10-18T05:10:37.032736Z","shell.execute_reply.started":"2023-10-18T05:10:36.992890Z","shell.execute_reply":"2023-10-18T05:10:37.031878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n** you found this notebook helpful or you just liked it , some upvotes would be very much appreciated - That will keep me motivated :)**\n","metadata":{"_uuid":"0ec3bcf47cd49afea61aa9ed60ced64de8fc2d3d","_cell_guid":"b1c2790e-4313-4b47-937a-b4eaaae6d845"}}]}