{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Table of Content\n\n* **Data preparation**\n    * Load dataset\n    * Split dataset\n    \n* **Deep Learning - CNN**\n    * Data augmentation\n    * Create CNN model\n    * Model training\n\n* **Evaluate**\n    * History chart\n    * Confusion Matrix\n    * Performance test\n\n* **Submission**\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-05T12:07:32.904974Z","iopub.execute_input":"2022-05-05T12:07:32.906279Z","iopub.status.idle":"2022-05-05T12:07:32.93959Z","shell.execute_reply.started":"2022-05-05T12:07:32.90611Z","shell.execute_reply":"2022-05-05T12:07:32.938201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Preparation","metadata":{}},{"cell_type":"markdown","source":"## Load datasets","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('../input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('../input/digit-recognizer/test.csv')\nsubmission_data = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:07:37.303399Z","iopub.execute_input":"2022-05-05T12:07:37.304188Z","iopub.status.idle":"2022-05-05T12:07:42.156197Z","shell.execute_reply.started":"2022-05-05T12:07:37.304119Z","shell.execute_reply":"2022-05-05T12:07:42.155542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"train.csv size is {train_data.shape}\")\nprint(f\"test.csv size is {test_data.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:07:49.69132Z","iopub.execute_input":"2022-05-05T12:07:49.6916Z","iopub.status.idle":"2022-05-05T12:07:49.697437Z","shell.execute_reply.started":"2022-05-05T12:07:49.691569Z","shell.execute_reply":"2022-05-05T12:07:49.696545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_data.drop(['label'], axis=1).values\ny = train_data['label'].values","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:07:52.488491Z","iopub.execute_input":"2022-05-05T12:07:52.48922Z","iopub.status.idle":"2022-05-05T12:07:52.612831Z","shell.execute_reply.started":"2022-05-05T12:07:52.489182Z","shell.execute_reply":"2022-05-05T12:07:52.611929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\n\nX = X.reshape(-1,28,28,1)\ny = to_categorical(y)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T12:07:54.527587Z","iopub.execute_input":"2022-05-05T12:07:54.528335Z","iopub.status.idle":"2022-05-05T12:08:00.594338Z","shell.execute_reply.started":"2022-05-05T12:07:54.528295Z","shell.execute_reply":"2022-05-05T12:08:00.593224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split dataset","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=42) #ratio 90:10","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.05122Z","iopub.execute_input":"2022-05-05T07:14:15.051498Z","iopub.status.idle":"2022-05-05T07:14:15.43385Z","shell.execute_reply.started":"2022-05-05T07:14:15.051449Z","shell.execute_reply":"2022-05-05T07:14:15.433115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Training data size is {X_train.shape}\")\nprint(f\"Testing data size is {X_test.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.43528Z","iopub.execute_input":"2022-05-05T07:14:15.435535Z","iopub.status.idle":"2022-05-05T07:14:15.43995Z","shell.execute_reply.started":"2022-05-05T07:14:15.435502Z","shell.execute_reply":"2022-05-05T07:14:15.439274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deep Learning - CNN","metadata":{}},{"cell_type":"markdown","source":"## Data augmentation","metadata":{}},{"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n    rotation_range=5,\n    width_shift_range=0.05,\n    height_shift_range=0.05,\n    zoom_range=0.05,\n    fill_mode='nearest',\n    rescale=1./255,\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.44149Z","iopub.execute_input":"2022-05-05T07:14:15.441958Z","iopub.status.idle":"2022-05-05T07:14:15.451537Z","shell.execute_reply.started":"2022-05-05T07:14:15.441922Z","shell.execute_reply":"2022-05-05T07:14:15.450535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_gen = datagen.flow(X_train, y_train, batch_size = 64)\ntest_gen = datagen.flow(X_test, y_test, batch_size = 64)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.453843Z","iopub.execute_input":"2022-05-05T07:14:15.454028Z","iopub.status.idle":"2022-05-05T07:14:15.517396Z","shell.execute_reply.started":"2022-05-05T07:14:15.454005Z","shell.execute_reply":"2022-05-05T07:14:15.516577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create CNN model","metadata":{}},{"cell_type":"code","source":"import keras\n\nmodel = keras.models.Sequential([\n    keras.layers.Conv2D(16, (3,3), padding = 'same', activation='relu', input_shape=(28, 28, 1)),\n    keras.layers.Conv2D(32, (3,3), padding = 'same', activation='relu'),\n    keras.layers.MaxPooling2D(2, 2),\n    keras.layers.Dropout(0.2),\n\n    keras.layers.Conv2D(32, (3,3), padding = 'same', activation='relu'),\n    keras.layers.Conv2D(64, (3,3), padding = 'same', activation='relu'),\n    keras.layers.MaxPooling2D(2, 2),\n    keras.layers.Dropout(0.2),\n    \n    keras.layers.Conv2D(64, (3,3), padding = 'same', activation='relu'),\n    keras.layers.Conv2D(128, (3,3), padding = 'same', activation='relu'),\n    keras.layers.MaxPooling2D(2, 2),\n    keras.layers.Dropout(0.2),\n\n    keras.layers.Conv2D(128, (3,3), padding = 'same', activation='relu'),\n    keras.layers.Conv2D(256, (3,3), padding = 'same', activation='relu'),\n    keras.layers.MaxPooling2D(2, 2),\n    keras.layers.Dropout(0.2),\n\n    keras.layers.Flatten(),\n    keras.layers.Dense(512, activation='relu'),\n    keras.layers.BatchNormalization(),\n    keras.layers.Dropout(0.5),\n    keras.layers.Dense(10, activation='softmax')\n])\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.518882Z","iopub.execute_input":"2022-05-05T07:14:15.519129Z","iopub.status.idle":"2022-05-05T07:14:15.660348Z","shell.execute_reply.started":"2022-05-05T07:14:15.519093Z","shell.execute_reply":"2022-05-05T07:14:15.659625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\nmodel.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.663131Z","iopub.execute_input":"2022-05-05T07:14:15.663467Z","iopub.status.idle":"2022-05-05T07:14:15.675007Z","shell.execute_reply.started":"2022-05-05T07:14:15.663431Z","shell.execute_reply":"2022-05-05T07:14:15.674225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model training","metadata":{}},{"cell_type":"code","source":"from keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\nes = EarlyStopping(\n    monitor='val_accuracy',\n    min_delta=0.0001,\n    patience=10,\n    verbose=1,\n    mode='max',\n    restore_best_weights=True\n)\n\nlr = ReduceLROnPlateau(\n    monitor='val_accuracy',\n    factor=0.2,\n    patience=5,\n    verbose=1,\n    mode='max',\n    min_delta=0.0001,\n    min_lr=0.000001\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.678063Z","iopub.execute_input":"2022-05-05T07:14:15.678283Z","iopub.status.idle":"2022-05-05T07:14:15.686042Z","shell.execute_reply.started":"2022-05-05T07:14:15.678258Z","shell.execute_reply":"2022-05-05T07:14:15.685283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\ntrain_steps = len(X_train) / batch_size\ntest_steps = len(X_test) / batch_size\n\nhistory = model.fit(\n    train_gen,\n    validation_data = test_gen,\n    epochs = 100, \n    steps_per_epoch = train_steps,\n    validation_steps = test_steps,\n    callbacks=[es, lr],\n    verbose=1\n)\n\nprint(\"\\n[INFO] Train Accuracy:\", \"{:.5f}%\".format(history.history['accuracy'][-1]*100))\nprint(\"[INFO] Train Loss    :\", \"{:.7f}\".format(history.history['loss'][-1]))\nprint(\"[INFO] Test Accuracy  :\", \"{:.5f}%\".format(history.history['val_accuracy'][-1]*100))\nprint(\"[INFO] Test Loss      :\", \"{:.7f}\".format(history.history['val_loss'][-1]))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:14:15.68829Z","iopub.execute_input":"2022-05-05T07:14:15.689064Z","iopub.status.idle":"2022-05-05T07:21:48.488549Z","shell.execute_reply.started":"2022-05-05T07:14:15.689026Z","shell.execute_reply":"2022-05-05T07:21:48.487175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate","metadata":{}},{"cell_type":"code","source":"evaluate_test = model.evaluate(X_test, y_test, verbose=1)\n\nprint(\"\\nAccuracy =\", \"{:.7f}%\".format(evaluate_test[1]*100))\nprint(\"Loss     =\" ,\"{:.9f}\".format(evaluate_test[0]))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:48.49029Z","iopub.execute_input":"2022-05-05T07:21:48.490561Z","iopub.status.idle":"2022-05-05T07:21:49.293806Z","shell.execute_reply.started":"2022-05-05T07:21:48.490525Z","shell.execute_reply":"2022-05-05T07:21:49.293098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## History chart","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model Accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='lower right')\nplt.show()\n\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model Loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'validation'], loc='upper right')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:49.295237Z","iopub.execute_input":"2022-05-05T07:21:49.295501Z","iopub.status.idle":"2022-05-05T07:21:49.645897Z","shell.execute_reply.started":"2022-05-05T07:21:49.295466Z","shell.execute_reply":"2022-05-05T07:21:49.645247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Confusion matrix","metadata":{}},{"cell_type":"code","source":"new_X_test = X_test.reshape(len(X_test),28,28,1) / 255.0\nnew_y_test = np.argmax(y_test, axis=-1)\nyhat_test = np.argmax(model.predict(X_test), axis=-1)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:49.646986Z","iopub.execute_input":"2022-05-05T07:21:49.647753Z","iopub.status.idle":"2022-05-05T07:21:50.032122Z","shell.execute_reply.started":"2022-05-05T07:21:49.647672Z","shell.execute_reply":"2022-05-05T07:21:50.031353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sn\nfrom sklearn.metrics import confusion_matrix\n\nconf_matrix = confusion_matrix(new_y_test, yhat_test)\ndf_cm = pd.DataFrame(conf_matrix,\n                     index = range(0,10),\n                     columns = range(0,10))\nplt.figure(figsize = (8,6)) #ratio 4:3\nsn.heatmap(df_cm, cmap=\"YlGnBu\", fmt=\"d\", cbar=False, annot=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:50.03359Z","iopub.execute_input":"2022-05-05T07:21:50.033848Z","iopub.status.idle":"2022-05-05T07:21:50.550794Z","shell.execute_reply.started":"2022-05-05T07:21:50.033811Z","shell.execute_reply":"2022-05-05T07:21:50.550121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Performance test","metadata":{}},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,20)) #ratio 3:4\nfor pick_random in range(1,26):\n    roll_number = np.random.randint(0, len(X_test)-1)\n    output_result = (new_y_test[roll_number] == yhat_test[roll_number])\n    \n    fig.add_subplot(5, 5, pick_random)\n    plt.title(f\"Real: {new_y_test[roll_number]}, Predicted: {yhat_test[roll_number]} \\nResult: {output_result}\")\n    plt.imshow(new_X_test[roll_number], cmap=\"binary\")","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:50.55198Z","iopub.execute_input":"2022-05-05T07:21:50.552259Z","iopub.status.idle":"2022-05-05T07:21:53.357889Z","shell.execute_reply.started":"2022-05-05T07:21:50.55221Z","shell.execute_reply":"2022-05-05T07:21:53.357251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission_label = np.argmax(model.predict(test_data.values.reshape(-1,28,28,1)), axis=-1)\nsubmission_label = pd.Series(submission_label, name=\"Label\")\n\nimage_id = pd.Series(range(1,len(test_data)+1))\nimage_id = pd.Series(image_id, name=\"ImageId\")","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:53.359131Z","iopub.execute_input":"2022-05-05T07:21:53.361443Z","iopub.status.idle":"2022-05-05T07:21:56.406973Z","shell.execute_reply.started":"2022-05-05T07:21:53.3614Z","shell.execute_reply":"2022-05-05T07:21:56.406189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([image_id,submission_label],axis = 1)\nsubmission.to_csv(\"digit_recognizer_submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T07:21:56.408069Z","iopub.execute_input":"2022-05-05T07:21:56.408313Z","iopub.status.idle":"2022-05-05T07:21:56.4556Z","shell.execute_reply.started":"2022-05-05T07:21:56.40828Z","shell.execute_reply":"2022-05-05T07:21:56.454974Z"},"trusted":true},"execution_count":null,"outputs":[]}]}