{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Digit Recognizer — CNN with Data Augmentation","metadata":{}},{"cell_type":"markdown","source":"This notebook is meant to document my learning of techniques like CNNs and data augmentation. The CNN architecture is adapted from a well-written and comprehensive kernel shared by the author [here](https://www.kaggle.com/code/cdeotte/how-to-choose-cnn-architecture-mnist).","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.callbacks import LearningRateScheduler\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Conv2D, Flatten, Dense, BatchNormalization, Dropout\n\nsns.set_theme(style=\"darkgrid\", font_scale=1.2)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# 1. Loading and preprocessing data.","metadata":{}},{"cell_type":"code","source":"X_train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\nX_test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ny_train = X_train.pop(\"label\").values\n\n# rescale to range [0, 1]\nX_train = X_train / 255.\nX_test = X_test / 255.\n\n# reshape to (samples, height, width, channels)\nX_train = X_train.values.reshape(-1, 28, 28, 1)\nX_test = X_test.values.reshape(-1, 28, 28, 1)\n\n# split training data into training and validation sets\nX_train, X_valid, y_train, y_valid = train_test_split(\n    X_train, y_train,\n    test_size=0.1,\n    shuffle=True,\n    stratify=y_train,\n    random_state=1343,\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Understanding image augmentation.","metadata":{}},{"cell_type":"code","source":"def augment_image(X, y, **kwargs):\n    X = X.copy()\n    y = y.copy()\n    \n    # shuffle datasets to view different images\n    shuffle_idx = np.random.permutation(len(y))\n    X = X[shuffle_idx]\n    y = y[shuffle_idx]\n    datagen = ImageDataGenerator(**kwargs)\n    \n    for X_, y_ in datagen.flow(\n        X, y,\n        batch_size=10,\n        shuffle=False,\n    ):\n        fig1 = plt.figure(figsize=(18, 3))\n        plt.suptitle(\"BEFORE:\")\n        fig2 = plt.figure(figsize=(18, 3))\n        plt.suptitle(\"AFTER:\")\n        for i in range(10):\n            ax1 = fig1.add_subplot(1, 10, i+1)\n            ax1.imshow(X[i], cmap=\"gray\")\n            ax1.set_title(f\"Label: {y[i]}\")\n            ax1.set_xticks([])\n            ax1.set_yticks([])\n\n            ax2 = fig2.add_subplot(1, 10, i+1)\n            ax2.imshow(X_[i], cmap=\"gray\")\n            ax2.set_title(f\"Label: {y_[i]}\")\n            ax2.set_xticks([])\n            ax2.set_yticks([])\n        break\n\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image rotation.","metadata":{}},{"cell_type":"code","source":"augment_image(X_train, y_train, rotation_range=90)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image shearing.","metadata":{}},{"cell_type":"code","source":"augment_image(X_train, y_train, shear_range=60)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image zooming.","metadata":{}},{"cell_type":"code","source":"augment_image(X_train, y_train, zoom_range=0.7)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Image shifting.","metadata":{}},{"cell_type":"code","source":"augment_image(X_train, y_train, width_shift_range=0.5, height_shift_range=0.5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Building CNN architecture.","metadata":{}},{"cell_type":"code","source":"# Adapted from https://www.kaggle.com/code/cdeotte/how-to-choose-cnn-architecture-mnist\ndef build_model():\n    model = Sequential([\n        Conv2D(32, kernel_size=3, activation=\"relu\", input_shape=(28, 28, 1)),\n        BatchNormalization(),\n        Conv2D(32, kernel_size=3, activation=\"relu\"),\n        BatchNormalization(),\n        Conv2D(32, kernel_size=5, activation=\"relu\", strides=2, padding=\"same\"),\n        BatchNormalization(),\n        Dropout(0.4),\n        \n        Conv2D(64, kernel_size=3, activation=\"relu\"),\n        BatchNormalization(),\n        Conv2D(64, kernel_size=3, activation=\"relu\"),\n        BatchNormalization(),\n        Conv2D(64, kernel_size=5, activation=\"relu\", strides=2, padding=\"same\"),\n        BatchNormalization(),\n        Dropout(0.4),\n        \n        Flatten(),\n        Dense(128, activation=\"relu\"),\n        BatchNormalization(),\n        Dropout(0.4),\n        Dense(10, activation=\"softmax\"),\n    ])\n\n    model.compile(\n        optimizer=\"adam\",\n        loss=\"sparse_categorical_crossentropy\",\n        metrics=\"accuracy\",\n    ) \n\n    return model  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Preliminaries to model training.","metadata":{}},{"cell_type":"code","source":"# Image augmentation\ndatagen = ImageDataGenerator(\n    rotation_range=10,\n    shear_range=10,\n    zoom_range=0.1,\n    width_shift_range=0.1,\n    height_shift_range=0.1,\n)\n\n# Exponential decay learning rate schedule\nlr_schedule = LearningRateScheduler(lambda epoch: 1e-3 * (0.95**epoch))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Training model.","metadata":{}},{"cell_type":"code","source":"model = build_model()\nhistory = model.fit(\n    datagen.flow(X_train, y_train, batch_size=64),\n    epochs=50,\n    callbacks=[lr_schedule],\n    validation_data=(X_valid, y_valid),\n).history","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Learning curves.","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(18, 4.5))\n\nplt.subplot(121)\nplt.plot(history[\"loss\"], label = \"Training\")\nplt.plot(history[\"val_loss\"], label = \"Validation\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Loss Curves\")\nplt.legend(loc=\"upper right\")\n\nplt.subplot(122)\nplt.plot(history[\"accuracy\"], label = \"Training\")\nplt.plot(history[\"val_accuracy\"], label = \"Validation\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.title(\"Accuracy Curves\")\nplt.legend(loc=\"lower right\")\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Incorrect predictions on validation data.","metadata":{}},{"cell_type":"code","source":"valid_pred = np.argmax(model.predict(X_valid), axis=1)\n\nplt.figure(figsize=(18, 6))\nplt.suptitle(f\"{np.sum(y_valid!=valid_pred)} out of {len(valid_pred)} images misclassified:\")\n\nfor i in range(np.sum(y_valid!=valid_pred)):      \n    plt.subplot((np.sum(y_valid!=valid_pred)//10)+1, 10, i+1)\n    plt.imshow(X_valid[y_valid!=valid_pred][i], cmap=\"gray\")\n    plt.title(f\"True Label: {y_valid[y_valid!=valid_pred][i]}\\nPrediction: {valid_pred[y_valid!=valid_pred][i]}\")\n    plt.xticks([])\n    plt.yticks([])\n    \nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 8. Further training on validation data.","metadata":{}},{"cell_type":"code","source":"_ = model.fit(\n    datagen.flow(X_valid, y_valid, batch_size=64),\n    epochs=50,\n    callbacks=[lr_schedule],\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 9. Submission.","metadata":{}},{"cell_type":"code","source":"final_pred = np.argmax(model.predict(X_test), axis=1)\nfinal_pred = pd.DataFrame({\n    \"ImageId\": range(1, len(final_pred)+1),\n    \"Label\": final_pred,\n})\n\nfinal_pred.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pred.to_csv(\"final_pred.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}}]}