{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823,"mountSlug":"competitions/digit-recognizer"}],"dockerImageVersionId":31400,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Welcome to the Digit Recognizer Competition\n#### Let's implement an ML model to identify written digits from 0 to 9\n#### Kaggle link to the competition : https://www.kaggle.com/competitions/digit-recognizer","metadata":{}},{"cell_type":"code","source":"# Let's import all librairies needed\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport kagglehub\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nimport random\nimport keras_tuner as kt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:04:49.781648Z","iopub.execute_input":"2026-06-15T16:04:49.782087Z","iopub.status.idle":"2026-06-15T16:05:15.340011Z","shell.execute_reply.started":"2026-06-15T16:04:49.782053Z","shell.execute_reply":"2026-06-15T16:05:15.339091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 01: Exploratory Data Analysis \n#### Let's import the data, normalize it, and plot a few examples","metadata":{}},{"cell_type":"code","source":"# Let's import the data and print the first few lines\nunprocessed_data = pd.read_csv(\"/kaggle/input/competitions/digit-recognizer/train.csv\", dtype='float64')\nunprocessed_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:15.341914Z","iopub.execute_input":"2026-06-15T16:05:15.342482Z","iopub.status.idle":"2026-06-15T16:05:19.076901Z","shell.execute_reply.started":"2026-06-15T16:05:15.342454Z","shell.execute_reply":"2026-06-15T16:05:19.075692Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's plot some of those digits\nN = random.randint(1, unprocessed_data.shape[0]-9)\nplt.figure(figsize=(5,5))\nfor i in range(9):\n    plt.subplot(3,3,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(unprocessed_data.iloc[i+N,1:].to_numpy().reshape(28, 28), cmap='binary')\n    plt.xlabel(str(unprocessed_data.iloc[i+N,0]))\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:19.078118Z","iopub.execute_input":"2026-06-15T16:05:19.078431Z","iopub.status.idle":"2026-06-15T16:05:19.378911Z","shell.execute_reply.started":"2026-06-15T16:05:19.078404Z","shell.execute_reply":"2026-06-15T16:05:19.377956Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 02: Processing and creating the different datasets\n#### Let's split the data into the train, cross-validation and test datasets\n","metadata":{}},{"cell_type":"code","source":"#Now let's create our train set, cross-validation set and test set\ntrain_dataset, test_dataset = train_test_split(\n    unprocessed_data, \n    test_size=0.1, \n    random_state=42, \n    stratify=unprocessed_data['label']  \n)\ntraining_dataset, cross_validation_dataset = train_test_split(\n    train_dataset, \n    test_size=0.1, \n    random_state=42, \n    stratify=train_dataset['label']  \n)\nX_test = test_dataset.iloc[:,1:].to_numpy()\ny_test = test_dataset.iloc[:,0].to_numpy()\nX_train = training_dataset.iloc[:,1:].to_numpy()\ny_train = training_dataset.iloc[:,0].to_numpy()\nX_cv = cross_validation_dataset.iloc[:,1:].to_numpy()\ny_cv = cross_validation_dataset.iloc[:,0].to_numpy()\nprint(f\"Train : {X_train.shape} | Cross Validation : {X_cv.shape} | Test : {X_test.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:19.379918Z","iopub.execute_input":"2026-06-15T16:05:19.380199Z","iopub.status.idle":"2026-06-15T16:05:19.839617Z","shell.execute_reply.started":"2026-06-15T16:05:19.380176Z","shell.execute_reply":"2026-06-15T16:05:19.838649Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's plot the different dataset and the % of each label in the dataset (in order to verify that each label is equally represented)\nfig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(15, 5))\n\ncounts = training_dataset['label'].value_counts().sort_index()\ncounts_ratio = training_dataset['label'].value_counts(normalize=True).round(3).sort_index()\ncounts.plot.bar(ax=ax1, color='steelblue', edgecolor='black')\nax1.bar_label(ax1.containers[0],labels=[f'{v:.1%}' for v in counts_ratio],padding=3)\nax1.set_xlabel(\"Labels\")\nax1.set_ylabel(\"Count\")\nax1.set_title(\"Labels distribution (training set)\")\nax1.tick_params(axis='x', rotation=0)\nax1.set_ylim(0, 4000)\n\ncounts = cross_validation_dataset['label'].value_counts().sort_index()\ncounts.plot.bar(ax=ax2, color='steelblue', edgecolor='black')\ncounts_ratio = cross_validation_dataset['label'].value_counts(normalize=True).round(3).sort_index()\nax2.bar_label(ax2.containers[0],labels=[f'{v:.1%}' for v in counts_ratio],padding=3)\nax2.set_xlabel(\"Labels\")\nax2.set_ylabel(\"Count\")\nax2.set_title(\"Labels distribution (cross-validation set)\")\nax2.tick_params(axis='x', rotation=0)\nax2.set_ylim(0, 4000)\n\ncounts = test_dataset['label'].value_counts().sort_index()\ncounts.plot.bar(ax=ax3, color='steelblue', edgecolor='black')\ncounts_ratio = test_dataset['label'].value_counts(normalize=True).round(3).sort_index()\nax3.bar_label(ax3.containers[0],labels=[f'{v:.1%}' for v in counts_ratio],padding=3)\nax3.set_xlabel(\"Labels\")\nax3.set_ylabel(\"Count\")\nax3.set_title(\"Labels distribution (test set)\")\nax3.tick_params(axis='x', rotation=0)\nax3.set_ylim(0, 4000)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:19.840578Z","iopub.execute_input":"2026-06-15T16:05:19.840847Z","iopub.status.idle":"2026-06-15T16:05:20.554288Z","shell.execute_reply.started":"2026-06-15T16:05:19.840822Z","shell.execute_reply":"2026-06-15T16:05:20.553362Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 03: Definition of the model and training\n#### Let's define the model and analyse the outputs on the training and test set","metadata":{}},{"cell_type":"code","source":"# Let's create a layer to normalize the data and add it to the model\nNormalizer = tf.keras.layers.Normalization(axis=-1)\nNormalizer.adapt(X_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:20.555475Z","iopub.execute_input":"2026-06-15T16:05:20.555841Z","iopub.status.idle":"2026-06-15T16:05:20.793061Z","shell.execute_reply.started":"2026-06-15T16:05:20.555803Z","shell.execute_reply":"2026-06-15T16:05:20.792026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's create the model (each layers has been optimally chosen by Keras Tuner)\ndigit_recognizer_model = Sequential([\n    Normalizer,\n    Dense(672, activation='relu'),\n    Dense(10, activation='linear')\n])\n\ndigit_recognizer_model.compile(\n    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001),\n    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]\n)\n\ndigit_recognizer_model.fit(\n    X_train, y_train,\n    epochs=50,\n    validation_data=(X_cv, y_cv),\n    callbacks=[tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:05:20.796251Z","iopub.execute_input":"2026-06-15T16:05:20.796522Z","iopub.status.idle":"2026-06-15T16:06:19.636728Z","shell.execute_reply.started":"2026-06-15T16:05:20.796499Z","shell.execute_reply":"2026-06-15T16:06:19.635433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Now let's evaluate our model on all sets\ntrain_loss, train_acc = digit_recognizer_model.evaluate(X_train,  y_train)\ncv_loss, cv_acc = digit_recognizer_model.evaluate(X_cv,  y_cv)\ntest_loss, test_acc = digit_recognizer_model.evaluate(X_test,  y_test)\n\nprint(f'Training Error (Loss): {train_loss:.4f} | Cross Validation Error (Loss): {cv_loss:.4f} | Test Error (Loss): {test_loss:.4f}')\nprint(f'Training Error (Accuracy): {train_acc:.4f} | Cross Validation Error (Accuracy): {cv_acc:.4f} | Test Error (Accuracy): {test_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:06:19.638041Z","iopub.execute_input":"2026-06-15T16:06:19.638335Z","iopub.status.idle":"2026-06-15T16:06:23.425449Z","shell.execute_reply.started":"2026-06-15T16:06:19.638309Z","shell.execute_reply":"2026-06-15T16:06:23.424628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 04: Confusion matrix\n#### Let's print the confusion matrix, analyse the issue and input more data to correct the issue","metadata":{}},{"cell_type":"code","source":"# Using the model, let's predict the labels of the training set and print the confusing matrix\ny_pred = np.argmax(digit_recognizer_model.predict(X_train), axis=1)\ncm = confusion_matrix(y_train, y_pred)\nConfusionMatrixDisplay(cm).plot()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:06:23.426638Z","iopub.execute_input":"2026-06-15T16:06:23.427003Z","iopub.status.idle":"2026-06-15T16:06:26.683724Z","shell.execute_reply.started":"2026-06-15T16:06:23.426968Z","shell.execute_reply":"2026-06-15T16:06:26.682721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's find the indexes of the mislabelised indexes\nmisclassified_idx = np.where(y_pred != y_train)[0]\n\n# Let's plot some of the mislabelised digits\nsamples = random.sample(list(misclassified_idx), min(9, len(misclassified_idx)))\n\nplt.figure(figsize=(6, 6))\nfor i, idx in enumerate(samples):\n    plt.subplot(3, 3, i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(X_train[idx].reshape(28, 28), cmap='binary')\n    plt.xlabel(f\"True: {int(y_train[idx])} | Predicted: {y_pred[idx]}\")\n\nplt.suptitle(\"Examples of digits mislabelised\", fontsize=13)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:06:26.685206Z","iopub.execute_input":"2026-06-15T16:06:26.685645Z","iopub.status.idle":"2026-06-15T16:06:26.965185Z","shell.execute_reply.started":"2026-06-15T16:06:26.685604Z","shell.execute_reply":"2026-06-15T16:06:26.964324Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 05: Error analysis, and data augmentation\n#### So the model seems to be struggling on some digits, so let's take all digits from the training and cross-validation sets, pivot them at different angles and add them back to the training dataset","metadata":{}},{"cell_type":"code","source":"DataAugmentation = ImageDataGenerator(\n    rotation_range=45,        # rotation ±45°\n    zoom_range=0.2,           # zoom ±20%\n    width_shift_range=0.1,    # vertical offset\n    height_shift_range=0.1,   # horizontal offset\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:06:26.966306Z","iopub.execute_input":"2026-06-15T16:06:26.966661Z","iopub.status.idle":"2026-06-15T16:06:26.971493Z","shell.execute_reply.started":"2026-06-15T16:06:26.966626Z","shell.execute_reply":"2026-06-15T16:06:26.970494Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's create more data of the mislabelised data by pivoting them\nX_aug_list = []\ny_aug_list = []\n\nfor _ in range(5):\n    for X_batch, y_batch in DataAugmentation.flow(\n        X_train.reshape(-1, 28, 28, 1), y_train,\n        batch_size=len(X_train)\n    ):\n        X_aug_list.append(X_batch.reshape(-1, 784))\n        y_aug_list.append(y_batch)\n        break\n        \n# Let's create the new training set by concatenating the initial set and the new training set \nX_train_augmented = np.concatenate([X_train] + X_aug_list)\ny_train_augmented = np.concatenate([y_train] + y_aug_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:06:26.972681Z","iopub.execute_input":"2026-06-15T16:06:26.973118Z","iopub.status.idle":"2026-06-15T16:07:11.471701Z","shell.execute_reply.started":"2026-06-15T16:06:26.973078Z","shell.execute_reply":"2026-06-15T16:07:11.470607Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's train again the model on the augmented dataset\ndigit_recognizer_model.fit(\n    X_train_augmented, y_train_augmented,\n    epochs=20,\n    validation_data=(X_cv, y_cv)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:07:11.472733Z","iopub.execute_input":"2026-06-15T16:07:11.473040Z","iopub.status.idle":"2026-06-15T16:19:29.413236Z","shell.execute_reply.started":"2026-06-15T16:07:11.473004Z","shell.execute_reply":"2026-06-15T16:19:29.412277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Now let's evaluate our model on all sets\ntrain_loss, train_acc = digit_recognizer_model.evaluate(X_train,  y_train)\ncv_loss, cv_acc = digit_recognizer_model.evaluate(X_cv,  y_cv)\ntest_loss, test_acc = digit_recognizer_model.evaluate(X_test,  y_test)\n\nprint(f'Training Error (Loss): {train_loss:.4f} | Cross Validation Error (Loss): {cv_loss:.4f} | Test Error (Loss): {test_loss:.4f}')\nprint(f'Training Error (Accuracy): {train_acc:.4f} | Cross Validation Error (Accuracy): {cv_acc:.4f} | Test Error (Accuracy): {test_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:19:29.414550Z","iopub.execute_input":"2026-06-15T16:19:29.414969Z","iopub.status.idle":"2026-06-15T16:19:33.165236Z","shell.execute_reply.started":"2026-06-15T16:19:29.414890Z","shell.execute_reply":"2026-06-15T16:19:33.164390Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 06: Prediction and submision\n#### Let's now to predict the labels of the competition and submit the results","metadata":{}},{"cell_type":"code","source":"# In order to have the best model, let's train it on the entire dataset before applying it to the test set of the competition\nX_final = np.concatenate([X_train, X_cv, X_test])\ny_final = np.concatenate([y_train, y_cv, y_test])\n\ndigit_recognizer_model.fit(X_final, y_final, epochs=50,\n    callbacks=[tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:19:33.166347Z","iopub.execute_input":"2026-06-15T16:19:33.166678Z","iopub.status.idle":"2026-06-15T16:21:50.972381Z","shell.execute_reply.started":"2026-06-15T16:19:33.166641Z","shell.execute_reply":"2026-06-15T16:21:50.971303Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Let's open the test set\ntest_kaggle = pd.read_csv(\"/kaggle/input/competitions/digit-recognizer/test.csv\", dtype='float64')\n\n# Let's apply the model\ny_kaggle_pred = np.argmax(digit_recognizer_model.predict(test_kaggle.to_numpy()), axis=1)\n\n# Let's build the submission file\nsubmission = pd.DataFrame({\n    'ImageId': np.arange(1, len(y_kaggle_pred) + 1),\n    'Label': y_kaggle_pred\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nsubmission.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-15T16:21:50.973696Z","iopub.execute_input":"2026-06-15T16:21:50.974096Z","iopub.status.idle":"2026-06-15T16:21:55.617803Z","shell.execute_reply.started":"2026-06-15T16:21:50.974057Z","shell.execute_reply":"2026-06-15T16:21:55.616919Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 07: Results & Key Takeaways\n\n## Overall score on the competition: 97.8%\n\n### Model Performance\n\n| Dataset     | Accuracy | Loss   |\n|-------------|----------|--------|\n| Training    | 98.7%    | 0.13   |\n| Cross-Val   | 95.9%    | 0.76   |\n| Test        | 96.5%    | 0.66   |\n| Competition | 97.8%    | ------ |\n\n### What I learned building this\n\n**1. Data splitting strategy matters**\nRather than a simple train/test split, I used a 3-way split (train / cross-validation / test) with stratification to ensure each digit class was equally represented across all sets. This gave me a reliable signal for overfitting without touching the test set.\n\n**2. Keras Tuner for hyperparameter optimization**\nInstead of manually guessing the number of neurons, I used Keras Tuner to find the optimal architecture. It selected a single hidden layer of 672 units — simpler than expected, but it works.\n\n**3. Error-driven data augmentation**\nAfter analyzing the confusion matrix, I applied data augmentation (rotation, zoom, shifts) specifically to address the model's weaknesses. This iterative approach — train → analyze → improve — is closer to how ML works in practice.\n\n**4. Early stopping to prevent overfitting**\nWith 50 epochs available, the model could easily overfit. Early stopping (patience=5 on val_loss) let the model train as long as it was learning, then stop automatically.\n\n### Tools used\n`TensorFlow / Keras` · `Keras Tuner` · `scikit-learn` · `pandas` · `matplotlib`","metadata":{}}]}