{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-22T04:27:24.598344Z","iopub.execute_input":"2023-06-22T04:27:24.598719Z","iopub.status.idle":"2023-06-22T04:27:24.604897Z","shell.execute_reply.started":"2023-06-22T04:27:24.598688Z","shell.execute_reply":"2023-06-22T04:27:24.603907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Dataset","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nprint(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:24.606561Z","iopub.execute_input":"2023-06-22T04:27:24.607495Z","iopub.status.idle":"2023-06-22T04:27:29.448869Z","shell.execute_reply.started":"2023-06-22T04:27:24.607462Z","shell.execute_reply":"2023-06-22T04:27:29.447842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simple EDA","metadata":{}},{"cell_type":"markdown","source":"## Label Distribution","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.450934Z","iopub.execute_input":"2023-06-22T04:27:29.451350Z","iopub.status.idle":"2023-06-22T04:27:29.457099Z","shell.execute_reply.started":"2023-06-22T04:27:29.451312Z","shell.execute_reply":"2023-06-22T04:27:29.456077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split Data\nX = train.drop('label',axis=1)\ny = train[['label']]","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.458542Z","iopub.execute_input":"2023-06-22T04:27:29.459168Z","iopub.status.idle":"2023-06-22T04:27:29.552989Z","shell.execute_reply.started":"2023-06-22T04:27:29.459134Z","shell.execute_reply":"2023-06-22T04:27:29.551712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(1,1, figsize=(12,8))\ny.label.value_counts().plot(kind='bar', rot=0, ax=ax)\nax.set_title('Train Dataset Label Count')\n# test.label.value_counts().plot(kind='bar', rot=0, ax=ax[1])\n# ax[1].set_title('Test Dataset Label Count')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.555979Z","iopub.execute_input":"2023-06-22T04:27:29.556373Z","iopub.status.idle":"2023-06-22T04:27:29.859705Z","shell.execute_reply.started":"2023-06-22T04:27:29.556334Z","shell.execute_reply":"2023-06-22T04:27:29.858782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.861278Z","iopub.execute_input":"2023-06-22T04:27:29.861913Z","iopub.status.idle":"2023-06-22T04:27:29.902342Z","shell.execute_reply.started":"2023-06-22T04:27:29.861878Z","shell.execute_reply":"2023-06-22T04:27:29.901355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.903840Z","iopub.execute_input":"2023-06-22T04:27:29.904458Z","iopub.status.idle":"2023-06-22T04:27:29.913485Z","shell.execute_reply.started":"2023-06-22T04:27:29.904420Z","shell.execute_reply":"2023-06-22T04:27:29.912517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reshape Data to 28x28\nBecause in dataset we have 784 pixel, so the image will be 28x28","metadata":{}},{"cell_type":"code","source":"X = X.values.reshape(-1, 28, 28, 1)\ntest = test.values.reshape(-1, 28, 28,1)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.915098Z","iopub.execute_input":"2023-06-22T04:27:29.915432Z","iopub.status.idle":"2023-06-22T04:27:29.924684Z","shell.execute_reply.started":"2023-06-22T04:27:29.915401Z","shell.execute_reply":"2023-06-22T04:27:29.923591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize Image Data","metadata":{}},{"cell_type":"code","source":"# Visualize some images\nf,ax = plt.subplots(2, 6, figsize=(12,8))\nfor i in range(6):\n    idx = random.randrange(len(X))\n    ax[0][i].imshow(X[idx], cmap='binary')\n    ax[0][i].set_xlabel(y.iloc[idx].values[0])\n    ax[1][i].imshow(X[idx+5], cmap='binary')\n    ax[1][i].set_xlabel(y.iloc[idx+5].values[0])\nplt.tight_layout() \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:29.926801Z","iopub.execute_input":"2023-06-22T04:27:29.927463Z","iopub.status.idle":"2023-06-22T04:27:31.573771Z","shell.execute_reply.started":"2023-06-22T04:27:29.927429Z","shell.execute_reply":"2023-06-22T04:27:31.572869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"markdown","source":"## Train - Val Split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:31.575349Z","iopub.execute_input":"2023-06-22T04:27:31.575922Z","iopub.status.idle":"2023-06-22T04:27:31.580352Z","shell.execute_reply.started":"2023-06-22T04:27:31.575887Z","shell.execute_reply":"2023-06-22T04:27:31.579472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, stratify=y, test_size=.2, random_state=1 )\nprint(\"X_train shape\",X_train.shape)\nprint(\"y_train shape\",y_train.shape)\nprint(\"X_val shape\",X_val.shape)\nprint(\"y_val shape\",y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:31.585267Z","iopub.execute_input":"2023-06-22T04:27:31.585600Z","iopub.status.idle":"2023-06-22T04:27:32.246877Z","shell.execute_reply.started":"2023-06-22T04:27:31.585575Z","shell.execute_reply":"2023-06-22T04:27:32.245758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Rescale Image","metadata":{}},{"cell_type":"code","source":"X_train = X_train / 255\nX_val = X_val / 255\nX_test = test / 255","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.248396Z","iopub.execute_input":"2023-06-22T04:27:32.249165Z","iopub.status.idle":"2023-06-22T04:27:32.429823Z","shell.execute_reply.started":"2023-06-22T04:27:32.249121Z","shell.execute_reply":"2023-06-22T04:27:32.428640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convert Image Label to Binary Matrix","metadata":{"execution":{"iopub.status.busy":"2023-05-30T03:50:28.858360Z","iopub.execute_input":"2023-05-30T03:50:28.858811Z","iopub.status.idle":"2023-05-30T03:50:28.899658Z","shell.execute_reply.started":"2023-05-30T03:50:28.858776Z","shell.execute_reply":"2023-05-30T03:50:28.897802Z"}}},{"cell_type":"code","source":"import tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.431496Z","iopub.execute_input":"2023-06-22T04:27:32.431962Z","iopub.status.idle":"2023-06-22T04:27:32.437141Z","shell.execute_reply.started":"2023-06-22T04:27:32.431904Z","shell.execute_reply":"2023-06-22T04:27:32.435984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = tf.keras.utils.to_categorical(y_train, num_classes=10)\ny_val = tf.keras.utils.to_categorical(y_val, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.438788Z","iopub.execute_input":"2023-06-22T04:27:32.439155Z","iopub.status.idle":"2023-06-22T04:27:32.450006Z","shell.execute_reply.started":"2023-06-22T04:27:32.439122Z","shell.execute_reply":"2023-06-22T04:27:32.449132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train data shape\nprint(X_train.shape, y_train.shape)\nprint(X_val.shape, y_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.451746Z","iopub.execute_input":"2023-06-22T04:27:32.452097Z","iopub.status.idle":"2023-06-22T04:27:32.462041Z","shell.execute_reply.started":"2023-06-22T04:27:32.452065Z","shell.execute_reply":"2023-06-22T04:27:32.460792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Image Augmentation","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.463537Z","iopub.execute_input":"2023-06-22T04:27:32.463894Z","iopub.status.idle":"2023-06-22T04:27:32.471771Z","shell.execute_reply.started":"2023-06-22T04:27:32.463861Z","shell.execute_reply":"2023-06-22T04:27:32.470830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagenerator = ImageDataGenerator(horizontal_flip=False,\n                                   vertical_flip=False,\n                                   shear_range=0.2,\n                                   zoom_range = 0.2,\n                                   width_shift_range=0.2,\n                                   height_shift_range=0.2,\n                                   rotation_range=10)\n\n# X_train2 = train_datagenerator.flow(x=X_train, y=y_train, batch_size=32)\ndatagenerator.fit(X_train)\n# X_test2 = train_datagenerator.flow(x=X_test, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.473383Z","iopub.execute_input":"2023-06-22T04:27:32.473726Z","iopub.status.idle":"2023-06-22T04:27:32.565486Z","shell.execute_reply.started":"2023-06-22T04:27:32.473694Z","shell.execute_reply":"2023-06-22T04:27:32.564369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN Model Training","metadata":{}},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.566907Z","iopub.execute_input":"2023-06-22T04:27:32.567406Z","iopub.status.idle":"2023-06-22T04:27:32.573972Z","shell.execute_reply.started":"2023-06-22T04:27:32.567371Z","shell.execute_reply":"2023-06-22T04:27:32.573025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_shape = (28, 28, 1)\nn_output = y_train.shape[1]\nmodel = tf.keras.Sequential([\n    # 1st Conv layer\n    tf.keras.layers.Conv2D(32, kernel_size=(3,3), padding='same', activation='relu', input_shape=input_shape),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Dropout(.25),\n    # 2nd Conv layer\n    tf.keras.layers.Conv2D(64, kernel_size=(3,3), padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Dropout(.25),\n    # 3nd Conv layer\n    tf.keras.layers.Conv2D(128, kernel_size=(3,3), padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Dropout(.25),\n    \n    # Classifier Head\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(32, activation='relu'),\n    tf.keras.layers.Dense(n_output, activation='softmax')\n])\nmodel.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.575516Z","iopub.execute_input":"2023-06-22T04:27:32.576179Z","iopub.status.idle":"2023-06-22T04:27:32.703956Z","shell.execute_reply.started":"2023-06-22T04:27:32.576144Z","shell.execute_reply":"2023-06-22T04:27:32.703209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define early stopping and LR reduction on training\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\nes = EarlyStopping(monitor = 'val_loss', mode = 'min', verbose = 1, patience = 3)\nlr_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.0001)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.705140Z","iopub.execute_input":"2023-06-22T04:27:32.705515Z","iopub.status.idle":"2023-06-22T04:27:32.712320Z","shell.execute_reply.started":"2023-06-22T04:27:32.705483Z","shell.execute_reply":"2023-06-22T04:27:32.711562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Training","metadata":{}},{"cell_type":"code","source":"# X_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.713546Z","iopub.execute_input":"2023-06-22T04:27:32.713890Z","iopub.status.idle":"2023-06-22T04:27:32.724034Z","shell.execute_reply.started":"2023-06-22T04:27:32.713857Z","shell.execute_reply":"2023-06-22T04:27:32.723192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# history = model.fit(X_train2, epochs=50, batch_size=32, callbacks=[es])\nhistory = model.fit(datagenerator.flow(X_train, y_train, batch_size=32), \n                              validation_data=(X_val,y_val),\n                              epochs=50,  \n                              callbacks=[es, lr_reduction])","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:27:32.725150Z","iopub.execute_input":"2023-06-22T04:27:32.725497Z","iopub.status.idle":"2023-06-22T04:32:43.427828Z","shell.execute_reply.started":"2023-06-22T04:27:32.725467Z","shell.execute_reply":"2023-06-22T04:32:43.426739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(1,2, figsize=(12,6))\nf.suptitle('Model Training Result')\nax[0].plot(history.history['loss'], label='train_loss',)\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].set_xlabel('epochs')\nax[0].set_ylabel('loss')\nax[0].set_title('Model Loss Result')\n\nax[1].plot(history.history['accuracy'], label='train_accuracy',)\nax[1].plot(history.history['val_accuracy'], label='val_accuracy')\nax[1].set_xlabel('epochs')\nax[1].set_ylabel('accuracy')\nax[1].set_title('Model Accuracy Result')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:43.429450Z","iopub.execute_input":"2023-06-22T04:32:43.429817Z","iopub.status.idle":"2023-06-22T04:32:44.028459Z","shell.execute_reply.started":"2023-06-22T04:32:43.429788Z","shell.execute_reply":"2023-06-22T04:32:44.027480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:44.029794Z","iopub.execute_input":"2023-06-22T04:32:44.030445Z","iopub.status.idle":"2023-06-22T04:32:46.525931Z","shell.execute_reply.started":"2023-06-22T04:32:44.030404Z","shell.execute_reply":"2023-06-22T04:32:46.524962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds = []\nfor i in range(len(y_pred)):\n    test_preds.append(np.argmax(y_pred[i]))","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:46.527346Z","iopub.execute_input":"2023-06-22T04:32:46.529212Z","iopub.status.idle":"2023-06-22T04:32:46.674455Z","shell.execute_reply.started":"2023-06-22T04:32:46.529176Z","shell.execute_reply":"2023-06-22T04:32:46.673351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test Visualization Result\nf,ax = plt.subplots(2, 7, figsize=(12,8))\nfor i in range(7):\n    idx = random.randrange(len(X_test))\n    ax[0][i].imshow(X_test[idx], cmap='binary')\n    ax[0][i].set_xlabel(test_preds[idx])\n    ax[1][i].imshow(X_test[idx+5], cmap='binary')\n    ax[1][i].set_xlabel(test_preds[idx+5])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:46.675967Z","iopub.execute_input":"2023-06-22T04:32:46.676648Z","iopub.status.idle":"2023-06-22T04:32:48.592749Z","shell.execute_reply.started":"2023-06-22T04:32:46.676610Z","shell.execute_reply":"2023-06-22T04:32:48.591850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({'ImageId': range(1, len(test_preds) + 1), 'Label': test_preds})","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:48.594235Z","iopub.execute_input":"2023-06-22T04:32:48.595273Z","iopub.status.idle":"2023-06-22T04:32:48.724727Z","shell.execute_reply.started":"2023-06-22T04:32:48.595237Z","shell.execute_reply":"2023-06-22T04:32:48.723273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the predictions to a CSV file\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-22T04:32:48.726350Z","iopub.execute_input":"2023-06-22T04:32:48.726821Z","iopub.status.idle":"2023-06-22T04:32:48.793063Z","shell.execute_reply.started":"2023-06-22T04:32:48.726784Z","shell.execute_reply":"2023-06-22T04:32:48.791978Z"},"trusted":true},"execution_count":null,"outputs":[]}]}