{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823}],"dockerImageVersionId":30132,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CNN + MNIST + Data Augmentation\nThe goal of this notebook is to classify, with the best accuracy possible handwritten digits.\nThe input is a `(28,28)` \"image\" in grey scale.\n\nThis notebook is using multiple technics to achieve 99.9 accuracy : \n* CNN\n* Denser Dataset (we use MNIST images)\n* Data Augmentation","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\nplt.rcParams['figure.figsize'] = [20, 20]\n\n# helper to show many images at once, for debug purpose\ndef show_images(images, labels, shape=(3,3)):\n    fig, p = plt.subplots(shape[0], shape[1])\n    i = 0\n    for x in p:\n        for ax in x:\n            ax.imshow(images[i])\n            ax.set_title(labels[i])\n            i += 1","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:39.469361Z","iopub.execute_input":"2026-03-25T18:29:39.469613Z","iopub.status.idle":"2026-03-25T18:29:44.512028Z","shell.execute_reply.started":"2026-03-25T18:29:39.469550Z","shell.execute_reply":"2026-03-25T18:29:44.511407Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# load the train and test data (csv files)\n# file structure: LABEL, PIXELS...\ntrain = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\") \n# file structure: PIXELS...\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\n\n# we reshape and normalize the data\ntrain_image = np.array(train.drop(['label'], axis=1), dtype=\"float32\") / 255\ntrain_image = train_image.reshape(-1, 28, 28, 1)\n\n# categorical transform 1 => [0 1 0 0 0 0], 3 => [0 0 0 1 0 0]...\ntrain_label = tf.keras.utils.to_categorical(train['label'])\n\ntest = np.array(test, dtype=\"float32\") / 255\ntest = test.reshape(-1, 28, 28, 1)\n\nshow_images(train_image[:25], train_label[:25], shape=(5,5))","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:44.513407Z","iopub.execute_input":"2026-03-25T18:29:44.513628Z","iopub.status.idle":"2026-03-25T18:29:50.988845Z","shell.execute_reply.started":"2026-03-25T18:29:44.513604Z","shell.execute_reply":"2026-03-25T18:29:50.988190Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Adding more data to the dataset\nThe dataset Kaggle provide is not the best. MNIST is a database of handwritten digit that can add data to our dataset.\n\nIn DL, the more data the better most of the time.\n\nSo the final dataset is Kaggle Data + MNIST.\n\nI choose to concatenate the training data and test data because we are using validation during the training process to track the evolution of the accuracy of the model.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.datasets import mnist\n(image_train_mnist, label_train_mnist), (image_test_mnist, label_test_mnist) = mnist.load_data()\nimage_mnist = np.concatenate((image_train_mnist, image_test_mnist))\nlabel_mnist = np.concatenate((label_train_mnist, label_test_mnist))\nimage_mnist = image_mnist.reshape(-1,28,28,1)\nimage_mnist = image_mnist.astype(np.float32) / 255\nlabel_mnist = tf.keras.utils.to_categorical(label_mnist,num_classes=10)\nimages = np.concatenate((train_image, image_mnist))\nlabels = np.concatenate((train_label, label_mnist))\n\n# final dataset shape\nprint(\"training image dataset shape:\", images.shape)\nprint(\"training label dataset shape:\", labels.shape)\n\nshow_images(images[:25], labels[:25], shape=(5,5))","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:50.991706Z","iopub.execute_input":"2026-03-25T18:29:50.991866Z","iopub.status.idle":"2026-03-25T18:29:54.165993Z","shell.execute_reply.started":"2026-03-25T18:29:50.991847Z","shell.execute_reply":"2026-03-25T18:29:54.165293Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data Augmentation\nTo provide more data during the training process, we are going to use Data Augmentation.\n\nData Augmentation is a fancy word to say that we are going the rotate, translate, zoom and shear image to create similar ones, but nevertheless differents ones.\n\nData Augmentation help the model to avoid overfitting","metadata":{}},{"cell_type":"code","source":"datagen = tf.keras.preprocessing.image.ImageDataGenerator(\n    rotation_range=20,\n    width_shift_range=0.20,\n    shear_range=15,\n    zoom_range=0.10,\n    validation_split=0.25,\n    horizontal_flip=False\n)\n\n# the train generator generate the images using the Data Augmentations rules defined in datagen\ntrain_generator = datagen.flow(\n    images,\n    labels, \n    batch_size=256,\n    subset='training',\n)\n\n# the validation generator generate the images using the Data Augmentations rules defined in datagen\nvalidation_generator = datagen.flow(\n    images,\n    labels, \n    batch_size=64,\n    subset='validation',\n)","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:54.167630Z","iopub.execute_input":"2026-03-25T18:29:54.167849Z","iopub.status.idle":"2026-03-25T18:29:54.224792Z","shell.execute_reply.started":"2026-03-25T18:29:54.167825Z","shell.execute_reply":"2026-03-25T18:29:54.224149Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## The model\nThe model is a very simple CNN. Nothing fancy here.\n\nWe use default optimizer and loss criteria.","metadata":{}},{"cell_type":"code","source":"def create_model():\n    model = tf.keras.Sequential([\n        tf.keras.layers.Reshape((28, 28, 1)),\n        tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), activation=\"relu\", padding=\"same\", input_shape=(28,28,1)),\n        tf.keras.layers.MaxPool2D((2,2)),\n        tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation=\"relu\", padding=\"same\"),\n        tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), activation=\"relu\", padding=\"same\"),\n        tf.keras.layers.MaxPool2D((2,2)),\n        tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), activation=\"relu\", padding=\"same\"),\n        tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), activation=\"relu\", padding=\"same\"),\n        tf.keras.layers.MaxPool2D((2,2)),\n\n        tf.keras.layers.Flatten(),\n        tf.keras.layers.Dense(512, activation=\"sigmoid\"),\n        tf.keras.layers.Dropout(0.25),\n        tf.keras.layers.Dense(512, activation=\"sigmoid\"),\n        tf.keras.layers.Dropout(0.25),\n        tf.keras.layers.Dense(256, activation=\"sigmoid\"),\n        tf.keras.layers.Dropout(0.1),\n        tf.keras.layers.Dense(10, activation=\"sigmoid\")\n    ])\n\n    model.compile(\n        optimizer=\"adam\", loss = 'categorical_crossentropy', metrics = ['accuracy']\n    )\n\n    return model\n\nmodel = create_model()","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:54.225767Z","iopub.execute_input":"2026-03-25T18:29:54.226013Z","iopub.status.idle":"2026-03-25T18:29:56.059705Z","shell.execute_reply.started":"2026-03-25T18:29:54.225982Z","shell.execute_reply":"2026-03-25T18:29:56.059091Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### ReduceLR and Checkpoint\nReduceLR is a way to tune the adam optimizer during training. If the val_loss metric start to become constant, the learning rate is decreased. That way we never stop learning. \n\nCheckpoint is to avoid loosing the best model if it is not the last epoch one.","metadata":{}},{"cell_type":"code","source":"reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',factor=0.1,patience=5 ,min_lr=0.000001,verbose=1)\ncheckpoint = tf.keras.callbacks.ModelCheckpoint(filepath='model.hdf5',monitor='val_loss',save_best_only=True,save_weights_only=True,verbose=1)","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:56.060617Z","iopub.execute_input":"2026-03-25T18:29:56.060819Z","iopub.status.idle":"2026-03-25T18:29:56.065147Z","shell.execute_reply.started":"2026-03-25T18:29:56.060798Z","shell.execute_reply":"2026-03-25T18:29:56.064450Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"We train the model over 60 epochs. Just trying differents number at this point.","metadata":{}},{"cell_type":"code","source":"history = model.fit_generator(train_generator, epochs=60, validation_data=validation_generator, callbacks=[reduce_lr,checkpoint], verbose=1)","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:29:56.066175Z","iopub.execute_input":"2026-03-25T18:29:56.066491Z","iopub.status.idle":"2026-03-25T18:53:19.085716Z","shell.execute_reply.started":"2026-03-25T18:29:56.066454Z","shell.execute_reply":"2026-03-25T18:53:19.084948Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_weights('model.hdf5')\n\nhistory_frame = pd.DataFrame(history.history)\nhistory_frame.loc[:, ['loss', 'val_loss']].plot()\nhistory_frame.loc[:, ['accuracy', 'val_accuracy']].plot();\n\ntest_loss, test_acc = model.evaluate(images[:500],  labels[:500], verbose=2)\nprint(\"model accuracy :\", test_acc, \", model loss\", test_loss)","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:53:19.086987Z","iopub.execute_input":"2026-03-25T18:53:19.087214Z","iopub.status.idle":"2026-03-25T18:53:19.951983Z","shell.execute_reply.started":"2026-03-25T18:53:19.087183Z","shell.execute_reply":"2026-03-25T18:53:19.951389Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Code used to submit the model to the competition\n\ndf = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\").astype(\"float32\") / 255.0\nres = tf.keras.backend.argmax(model.predict(df))\ncsv = pd.DataFrame({'ImageId': range(1, len(res) + 1), \"Label\": res})\ncsv.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2026-03-25T18:54:37.461023Z","iopub.execute_input":"2026-03-25T18:54:37.461741Z","iopub.status.idle":"2026-03-25T18:54:40.111275Z","shell.execute_reply.started":"2026-03-25T18:54:37.461707Z","shell.execute_reply":"2026-03-25T18:54:40.110701Z"},"trusted":true},"outputs":[],"execution_count":null}]}