{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":29837,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## About the Dataset\n> * MNIST (\"Modified National Institute of Standards and Technology\") is the de facto “hello world” dataset of computer vision.   \n> * Since its release in 1999, this classic dataset of handwritten images has served as the basis for benchmarking classification algorithms. \n> * As new machine learning techniques emerge, MNIST remains a reliable resource for researchers and learners alike.\n\n## Task\n> * To correctly identify digits from a dataset of tens of thousands of handwritten images in the test dataset","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"# for numerical analysis\nimport numpy as np \n# to store and process in a dataframe\nimport pandas as pd \n\n# for ploting graphs\nimport matplotlib.pyplot as plt\n# advancec ploting\nimport seaborn as sns\n\n# image processing\nimport matplotlib.image as mpimg\n\n# train test split\nfrom sklearn.model_selection import train_test_split\n# model performance metrics\nfrom sklearn.metrics import confusion_matrix, classification_report\n\n# utility functions\nfrom tensorflow.keras.utils import to_categorical\n# sequential model\nfrom tensorflow.keras.models import Sequential\n# layers\nfrom tensorflow.keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout, BatchNormalization\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-11T11:22:11.950491Z","iopub.execute_input":"2024-06-11T11:22:11.950845Z","iopub.status.idle":"2024-06-11T11:22:11.959197Z","shell.execute_reply.started":"2024-06-11T11:22:11.950793Z","shell.execute_reply":"2024-06-11T11:22:11.957766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data","metadata":{}},{"cell_type":"code","source":"# list of files\n! ls ../input/digit-recognizer","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:18.659596Z","iopub.execute_input":"2024-06-11T11:22:18.660005Z","iopub.status.idle":"2024-06-11T11:22:19.801913Z","shell.execute_reply.started":"2024-06-11T11:22:18.659934Z","shell.execute_reply":"2024-06-11T11:22:19.800694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import train and test dataset\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2024-06-11T11:22:24.098518Z","iopub.execute_input":"2024-06-11T11:22:24.098930Z","iopub.status.idle":"2024-06-11T11:22:30.162405Z","shell.execute_reply.started":"2024-06-11T11:22:24.098862Z","shell.execute_reply":"2024-06-11T11:22:30.161115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# training dataset\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:35.790541Z","iopub.execute_input":"2024-06-11T11:22:35.790954Z","iopub.status.idle":"2024-06-11T11:22:35.817523Z","shell.execute_reply.started":"2024-06-11T11:22:35.790879Z","shell.execute_reply":"2024-06-11T11:22:35.816582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train['label'].count())","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:40.899680Z","iopub.execute_input":"2024-06-11T11:22:40.900040Z","iopub.status.idle":"2024-06-11T11:22:40.907208Z","shell.execute_reply.started":"2024-06-11T11:22:40.899983Z","shell.execute_reply":"2024-06-11T11:22:40.906259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'].value_counts().sort_index()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:43.397899Z","iopub.execute_input":"2024-06-11T11:22:43.398305Z","iopub.status.idle":"2024-06-11T11:22:43.410256Z","shell.execute_reply.started":"2024-06-11T11:22:43.398247Z","shell.execute_reply":"2024-06-11T11:22:43.409247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test dataset\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:50.659903Z","iopub.execute_input":"2024-06-11T11:22:50.660326Z","iopub.status.idle":"2024-06-11T11:22:50.686185Z","shell.execute_reply.started":"2024-06-11T11:22:50.660260Z","shell.execute_reply":"2024-06-11T11:22:50.685203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# looking for missing values\nprint(train.isna().sum())\nprint(test.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:22:55.843425Z","iopub.execute_input":"2024-06-11T11:22:55.843799Z","iopub.status.idle":"2024-06-11T11:22:55.990596Z","shell.execute_reply.started":"2024-06-11T11:22:55.843746Z","shell.execute_reply":"2024-06-11T11:22:55.989508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"### Label count","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 5))\nsns.countplot(train['label'], palette='Dark2')\nplt.title('Train labels count')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:01.463651Z","iopub.execute_input":"2024-06-11T11:23:01.464034Z","iopub.status.idle":"2024-06-11T11:23:01.691678Z","shell.execute_reply.started":"2024-06-11T11:23:01.463982Z","shell.execute_reply":"2024-06-11T11:23:01.690608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first few train images with labels\nfig, ax = plt.subplots(figsize=(18, 8))\nfor ind, row in train.iloc[:8, :].iterrows():\n    plt.subplot(2, 4, ind+1)\n    plt.title(row[0])\n    img = row.to_numpy()[1:].reshape(28, 28)\n    fig.suptitle('Train images', fontsize=24)\n    plt.axis('off')\n    plt.imshow(img, cmap='magma')","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:06.711034Z","iopub.execute_input":"2024-06-11T11:23:06.711475Z","iopub.status.idle":"2024-06-11T11:23:07.425972Z","shell.execute_reply.started":"2024-06-11T11:23:06.711400Z","shell.execute_reply":"2024-06-11T11:23:07.425026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Test images doesn't have labels  \n> We need to create a model to predict them","metadata":{"_kg_hide-input":true}},{"cell_type":"code","source":"# first few test images\nfig, ax = plt.subplots(figsize=(18, 8))\nfor ind, row in test.iloc[:8, :].iterrows():\n    plt.subplot(2, 4, ind+1)\n    img = row.to_numpy()[:].reshape(28, 28)\n    fig.suptitle('Test images', fontsize=24)\n    plt.axis('off')\n    plt.imshow(img, cmap='magma')","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:13.000164Z","iopub.execute_input":"2024-06-11T11:23:13.000931Z","iopub.status.idle":"2024-06-11T11:23:13.733665Z","shell.execute_reply.started":"2024-06-11T11:23:13.000845Z","shell.execute_reply":"2024-06-11T11:23:13.732158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"# split into image and labels and convert to numpy array\nX = train.iloc[:, 1:].to_numpy()\ny = train['label'].to_numpy()\n\n# test dataset\ntest = test.iloc[:, :].to_numpy()\n\nfor i in [X, y, test]:\n    print(i.shape)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:20.423884Z","iopub.execute_input":"2024-06-11T11:23:20.424292Z","iopub.status.idle":"2024-06-11T11:23:20.433119Z","shell.execute_reply.started":"2024-06-11T11:23:20.424234Z","shell.execute_reply":"2024-06-11T11:23:20.431705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# normalize the data\n# ==================\n\nX = X / 255.0\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:33.063336Z","iopub.execute_input":"2024-06-11T11:23:33.063708Z","iopub.status.idle":"2024-06-11T11:23:33.537455Z","shell.execute_reply.started":"2024-06-11T11:23:33.063656Z","shell.execute_reply":"2024-06-11T11:23:33.536398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reshape dataset\n# ===============\n\n# shape of training and test dataset\nprint(X.shape)\nprint(test.shape)\n\n# reshape the dataframe to 3x3 matrix with 1 channel grey scale values\nX = X.reshape(-1,28,28,1)\ntest = test.reshape(-1,28,28,1)\n\n# shape of training and test dataset\nprint(X.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:40.755338Z","iopub.execute_input":"2024-06-11T11:23:40.755712Z","iopub.status.idle":"2024-06-11T11:23:40.764795Z","shell.execute_reply.started":"2024-06-11T11:23:40.755657Z","shell.execute_reply":"2024-06-11T11:23:40.763551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# one hot encode target\n# =====================\n\n# shape and values of target\nprint(y.shape)\nprint(y[0])\n\n# convert Y_train to categorical by one-hot-encoding\ny_enc = to_categorical(y, num_classes = 10)\n\n# shape and values of target\nprint(y_enc.shape)\nprint(y_enc[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:51.071523Z","iopub.execute_input":"2024-06-11T11:23:51.071887Z","iopub.status.idle":"2024-06-11T11:23:51.081525Z","shell.execute_reply.started":"2024-06-11T11:23:51.071832Z","shell.execute_reply":"2024-06-11T11:23:51.080250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train test split\n# ================\n\n# random seed\nrandom_seed = 2\n\n# train validation split\nX_train, X_val, y_train_enc, y_val_enc = train_test_split(X, y_enc, test_size=0.3)\n\n# shape\nfor i in [X_train, y_train_enc, X_val, y_val_enc]:\n    print(i.shape)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:23:57.462980Z","iopub.execute_input":"2024-06-11T11:23:57.463452Z","iopub.status.idle":"2024-06-11T11:23:58.338315Z","shell.execute_reply.started":"2024-06-11T11:23:57.463375Z","shell.execute_reply":"2024-06-11T11:23:58.336954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot images","metadata":{}},{"cell_type":"code","source":"g = plt.imshow(X_train[0][:,:,0])\nprint(y_train_enc[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:04.384901Z","iopub.execute_input":"2024-06-11T11:24:04.385310Z","iopub.status.idle":"2024-06-11T11:24:04.582762Z","shell.execute_reply.started":"2024-06-11T11:24:04.385254Z","shell.execute_reply":"2024-06-11T11:24:04.581665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = plt.imshow(X_train[9][:,:,0])\nprint(y_train_enc[9])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:10.977568Z","iopub.execute_input":"2024-06-11T11:24:10.977933Z","iopub.status.idle":"2024-06-11T11:24:11.172764Z","shell.execute_reply.started":"2024-06-11T11:24:10.977880Z","shell.execute_reply":"2024-06-11T11:24:11.171719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN","metadata":{}},{"cell_type":"markdown","source":"### Model parameters","metadata":{}},{"cell_type":"code","source":"INPUT_SHAPE = (28,28,1)\nOUTPUT_SHAPE = 10\nBATCH_SIZE = 256\nEPOCHS = 20\nVERBOSE = 2","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:17.572411Z","iopub.execute_input":"2024-06-11T11:24:17.572768Z","iopub.status.idle":"2024-06-11T11:24:17.578954Z","shell.execute_reply.started":"2024-06-11T11:24:17.572718Z","shell.execute_reply":"2024-06-11T11:24:17.577855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define CNN Model","metadata":{}},{"cell_type":"code","source":"model = Sequential()\n\n# First Convolutional Block\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu', input_shape=INPUT_SHAPE))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D((2, 2)))\n\n# Second Convolutional Block\nmodel.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D((2, 2)))\n\n# Third Convolutional Block\nmodel.add(Conv2D(256, kernel_size=(3, 3), activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(MaxPool2D((2, 2)))\n\nmodel.add(Flatten())\n\n# Fully Connected Layers\nmodel.add(Dense(256, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(128, activation='relu'))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(OUTPUT_SHAPE, activation='softmax'))\n\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:27.594241Z","iopub.execute_input":"2024-06-11T11:24:27.594583Z","iopub.status.idle":"2024-06-11T11:24:28.339855Z","shell.execute_reply.started":"2024-06-11T11:24:27.594532Z","shell.execute_reply":"2024-06-11T11:24:28.338586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Compile model","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer='adam', \n              loss='categorical_crossentropy', \n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:46.445644Z","iopub.execute_input":"2024-06-11T11:24:46.446007Z","iopub.status.idle":"2024-06-11T11:24:46.505004Z","shell.execute_reply.started":"2024-06-11T11:24:46.445955Z","shell.execute_reply":"2024-06-11T11:24:46.503957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model summary","metadata":{}},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:52.554201Z","iopub.execute_input":"2024-06-11T11:24:52.554566Z","iopub.status.idle":"2024-06-11T11:24:52.566699Z","shell.execute_reply.started":"2024-06-11T11:24:52.554514Z","shell.execute_reply":"2024-06-11T11:24:52.565585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model fitting","metadata":{}},{"cell_type":"code","source":"history = model.fit(X_train, y_train_enc,\n                    epochs=EPOCHS,\n                    batch_size=BATCH_SIZE,\n                    verbose=VERBOSE,\n                    validation_split=0.3)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:24:59.229246Z","iopub.execute_input":"2024-06-11T11:24:59.229615Z","iopub.status.idle":"2024-06-11T11:41:01.109607Z","shell.execute_reply.started":"2024-06-11T11:24:59.229562Z","shell.execute_reply":"2024-06-11T11:41:01.108315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Accurayc and loss","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(14, 5))\n\nplt.subplot(1, 2, 1)\nplt.plot(history.history['accuracy'], label='Training Accuracy')\nplt.plot(history.history['val_accuracy'], label='Validation Accuracy')\nplt.legend(loc='lower right')\nplt.title('Training and Validation Accuracy')\n\nplt.subplot(1, 2, 2)\nplt.plot(history.history['loss'], label='Training Loss')\nplt.plot(history.history['val_loss'], label='Validation Loss')\nplt.legend(loc='upper right')\nplt.title('Training and Validation Loss')\n\nplt.savefig('./foo.png')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:51:22.522999Z","iopub.execute_input":"2024-06-11T11:51:22.523405Z","iopub.status.idle":"2024-06-11T11:51:23.084545Z","shell.execute_reply.started":"2024-06-11T11:51:22.523348Z","shell.execute_reply":"2024-06-11T11:51:23.083449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluating on validationa dataset","metadata":{}},{"cell_type":"code","source":"# model loss and accuracy on validation set\nmodel.evaluate(X_val, y_val_enc, verbose=False)","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:51:33.405648Z","iopub.execute_input":"2024-06-11T11:51:33.405990Z","iopub.status.idle":"2024-06-11T11:51:40.196959Z","shell.execute_reply.started":"2024-06-11T11:51:33.405939Z","shell.execute_reply":"2024-06-11T11:51:40.196146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicted values\ny_pred_enc = model.predict(X_val)\n\n# actual\ny_act = [np.argmax(i) for i in y_val_enc]\n\n# decoding predicted values\ny_pred = [np.argmax(i) for i in y_pred_enc]\n\nprint(y_pred_enc[0])\nprint(y_pred[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:51:48.360442Z","iopub.execute_input":"2024-06-11T11:51:48.360856Z","iopub.status.idle":"2024-06-11T11:51:55.881610Z","shell.execute_reply.started":"2024-06-11T11:51:48.360784Z","shell.execute_reply":"2024-06-11T11:51:55.880337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_act, y_pred))","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:52:04.025883Z","iopub.execute_input":"2024-06-11T11:52:04.026280Z","iopub.status.idle":"2024-06-11T11:52:04.080066Z","shell.execute_reply.started":"2024-06-11T11:52:04.026220Z","shell.execute_reply":"2024-06-11T11:52:04.079037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(7, 7))\nsns.heatmap(confusion_matrix(y_act, y_pred), annot=True, \n            cbar=False, fmt='1d', cmap='Blues', ax=ax)\nax.set_title('Confusion Matrix', loc='left', fontsize=16)\nax.set_xlabel('Predicted')\nax.set_ylabel('Actual')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:52:27.168889Z","iopub.execute_input":"2024-06-11T11:52:27.169351Z","iopub.status.idle":"2024-06-11T11:52:27.753005Z","shell.execute_reply.started":"2024-06-11T11:52:27.169285Z","shell.execute_reply":"2024-06-11T11:52:27.751858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Predicting on test","metadata":{}},{"cell_type":"code","source":"# predicted values\ny_pred_enc = model.predict(test)\n\n# decoding predicted values\ny_pred = [np.argmax(i) for i in y_pred_enc]\n\nprint(y_pred_enc[0])\nprint(y_pred[0])","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:52:34.279377Z","iopub.execute_input":"2024-06-11T11:52:34.279719Z","iopub.status.idle":"2024-06-11T11:52:49.718802Z","shell.execute_reply.started":"2024-06-11T11:52:34.279668Z","shell.execute_reply":"2024-06-11T11:52:49.717876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predicted targets of each images\n# (labels above the images are predicted labels)\nfig, ax = plt.subplots(figsize=(18, 12))\nfor ind, row in enumerate(test[:15]):\n    plt.subplot(3, 5, ind+1)\n    plt.title(y_pred[ind])\n    img = row.reshape(28, 28)\n    fig.suptitle('Predicted values', fontsize=24)\n    plt.axis('off')\n    plt.imshow(img, cmap='cividis')","metadata":{"execution":{"iopub.status.busy":"2024-06-11T11:52:57.965636Z","iopub.execute_input":"2024-06-11T11:52:57.966016Z","iopub.status.idle":"2024-06-11T11:52:59.354354Z","shell.execute_reply.started":"2024-06-11T11:52:57.965947Z","shell.execute_reply":"2024-06-11T11:52:59.353282Z"},"trusted":true},"execution_count":null,"outputs":[]}]}