{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-29T16:36:41.695902Z","iopub.execute_input":"2023-06-29T16:36:41.696360Z","iopub.status.idle":"2023-06-29T16:36:41.707948Z","shell.execute_reply.started":"2023-06-29T16:36:41.696322Z","shell.execute_reply":"2023-06-29T16:36:41.706785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:41.709901Z","iopub.execute_input":"2023-06-29T16:36:41.711066Z","iopub.status.idle":"2023-06-29T16:36:41.725894Z","shell.execute_reply.started":"2023-06-29T16:36:41.711017Z","shell.execute_reply":"2023-06-29T16:36:41.724403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename == 'train.csv':\n            train_data = pd.read_csv(os.path.join(dirname, filename))\n        elif filename == 'test.csv':\n            test_data = pd.read_csv(os.path.join(dirname, filename))\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:41.727777Z","iopub.execute_input":"2023-06-29T16:36:41.728523Z","iopub.status.idle":"2023-06-29T16:36:48.131445Z","shell.execute_reply.started":"2023-06-29T16:36:41.728403Z","shell.execute_reply":"2023-06-29T16:36:48.130142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\nprint(test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:48.132795Z","iopub.execute_input":"2023-06-29T16:36:48.133181Z","iopub.status.idle":"2023-06-29T16:36:48.139479Z","shell.execute_reply.started":"2023-06-29T16:36:48.133143Z","shell.execute_reply":"2023-06-29T16:36:48.137953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:48.143228Z","iopub.execute_input":"2023-06-29T16:36:48.143623Z","iopub.status.idle":"2023-06-29T16:36:48.184864Z","shell.execute_reply.started":"2023-06-29T16:36:48.143584Z","shell.execute_reply":"2023-06-29T16:36:48.183461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test data set has 784 columns, each column represent value of one pixel. Meaning the picture is 28X28X1\n\nTrain data set has one more column (feature) and that is target variable","metadata":{}},{"cell_type":"markdown","source":"Currently each pixel value for training and test dataset is stored in saperate column\nWe will need to reshape the data -> Creating N*28*28*1 numpy ND array","metadata":{}},{"cell_type":"code","source":"# Split the data into features and target\nX = train_data.drop('label', axis=1)  # Features\ny = train_data['label']  # Target\nX=X/255.0\nX=X.values.reshape(-1,28,28,1)\n\nX_train, X_val, y_train, y_val = train_test_split(X,y, test_size=0.2, random_state=42,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:48.186519Z","iopub.execute_input":"2023-06-29T16:36:48.186994Z","iopub.status.idle":"2023-06-29T16:36:48.784350Z","shell.execute_reply.started":"2023-06-29T16:36:48.186956Z","shell.execute_reply":"2023-06-29T16:36:48.782924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets now visualize the training image","metadata":{}},{"cell_type":"code","source":"img = np.reshape(X[1], (28, 28))\nplt.imshow(img, cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:48.785663Z","iopub.execute_input":"2023-06-29T16:36:48.786017Z","iopub.status.idle":"2023-06-29T16:36:49.273430Z","shell.execute_reply.started":"2023-06-29T16:36:48.785981Z","shell.execute_reply":"2023-06-29T16:36:49.272144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that we have the training data in ND array form, we can create a simple CNN model and train it on given data","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal',input_shape=(28, 28, 1)))\nmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal'))\nmodel.add(MaxPool2D((2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))\n\nmodel.compile(loss=keras.losses.sparse_categorical_crossentropy,\n              optimizer=keras.optimizers.RMSprop(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:49.275047Z","iopub.execute_input":"2023-06-29T16:36:49.275431Z","iopub.status.idle":"2023-06-29T16:36:49.376672Z","shell.execute_reply.started":"2023-06-29T16:36:49.275394Z","shell.execute_reply":"2023-06-29T16:36:49.375487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=15,batch_size=128, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:36:49.378215Z","iopub.execute_input":"2023-06-29T16:36:49.378590Z","iopub.status.idle":"2023-06-29T16:46:12.525613Z","shell.execute_reply.started":"2023-06-29T16:36:49.378552Z","shell.execute_reply":"2023-06-29T16:46:12.524160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2 = Sequential()\nmodel_2.add(Conv2D(64, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))\nmodel_2.add(Conv2D(64, kernel_size=(3, 3),activation='relu'))\nmodel_2.add(MaxPool2D((2, 2)))\nmodel_2.add(Dropout(0.25))\n\nmodel_2.add(Flatten())\nmodel_2.add(Dense(256, activation = \"relu\"))\nmodel_2.add(Dropout(0.5))\nmodel_2.add(Dense(10, activation = \"softmax\"))\n\nmodel_2.compile(loss=keras.losses.sparse_categorical_crossentropy,\n              optimizer=keras.optimizers.RMSprop(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:46:12.527474Z","iopub.execute_input":"2023-06-29T16:46:12.527823Z","iopub.status.idle":"2023-06-29T16:46:12.635934Z","shell.execute_reply.started":"2023-06-29T16:46:12.527787Z","shell.execute_reply":"2023-06-29T16:46:12.634760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_2 = model_2.fit(X_train, y_train, epochs=15,batch_size=128, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T16:46:12.637330Z","iopub.execute_input":"2023-06-29T16:46:12.637676Z","iopub.status.idle":"2023-06-29T17:08:35.559627Z","shell.execute_reply.started":"2023-06-29T16:46:12.637642Z","shell.execute_reply":"2023-06-29T17:08:35.558518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_3 = Sequential()\nmodel_3.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))\nmodel_3.add(Conv2D(32, kernel_size=(3, 3),activation='relu'))\nmodel_3.add(MaxPool2D((2, 2)))\nmodel_3.add(BatchNormalization())\nmodel_3.add(Dropout(0.25))\n\nmodel_3.add(Conv2D(64, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))\nmodel_3.add(Conv2D(64, kernel_size=(3, 3),activation='relu'))\nmodel_3.add(MaxPool2D((2, 2)))\nmodel_3.add(BatchNormalization())\nmodel_3.add(Dropout(0.25))\n\nmodel_3.add(Flatten())\nmodel_3.add(Dense(256, activation = \"relu\"))\nmodel_3.add(BatchNormalization())\nmodel_3.add(Dropout(0.25))\n\nmodel_3.add(Dense(128, activation = \"relu\"))\nmodel_3.add(BatchNormalization())\nmodel_3.add(Dropout(0.5))\n\nmodel_3.add(Dense(64, activation = \"relu\"))\nmodel_3.add(BatchNormalization())\nmodel_3.add(Dropout(0.5))\n\nmodel_3.add(Dense(10, activation = \"softmax\"))\n\nmodel_3.compile(loss=keras.losses.sparse_categorical_crossentropy,\n              optimizer=keras.optimizers.RMSprop(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:08:35.561471Z","iopub.execute_input":"2023-06-29T17:08:35.561843Z","iopub.status.idle":"2023-06-29T17:08:35.827826Z","shell.execute_reply.started":"2023-06-29T17:08:35.561805Z","shell.execute_reply":"2023-06-29T17:08:35.826356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_3 = model_3.fit(X_train, y_train, epochs=30,batch_size=128, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:08:35.829900Z","iopub.execute_input":"2023-06-29T17:08:35.830320Z","iopub.status.idle":"2023-06-29T17:31:56.999974Z","shell.execute_reply.started":"2023-06-29T17:08:35.830279Z","shell.execute_reply":"2023-06-29T17:31:56.998619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets Test the classification of this model on Test Data Set","metadata":{"execution":{"iopub.status.busy":"2023-04-22T18:00:41.237970Z","iopub.execute_input":"2023-04-22T18:00:41.238413Z","iopub.status.idle":"2023-04-22T18:00:41.246948Z","shell.execute_reply.started":"2023-04-22T18:00:41.238375Z","shell.execute_reply":"2023-04-22T18:00:41.245436Z"}}},{"cell_type":"code","source":"# Split the data into features and target\nX_test = test_data\nX_test=X_test/255.0\nX_test=X_test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:31:57.005927Z","iopub.execute_input":"2023-06-29T17:31:57.006430Z","iopub.status.idle":"2023-06-29T17:31:57.058089Z","shell.execute_reply.started":"2023-06-29T17:31:57.006386Z","shell.execute_reply":"2023-06-29T17:31:57.057028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v_1 = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:31:57.059188Z","iopub.execute_input":"2023-06-29T17:31:57.059570Z","iopub.status.idle":"2023-06-29T17:32:06.441394Z","shell.execute_reply.started":"2023-06-29T17:31:57.059533Z","shell.execute_reply":"2023-06-29T17:32:06.440010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v_2 = model_2.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:06.444552Z","iopub.execute_input":"2023-06-29T17:32:06.445079Z","iopub.status.idle":"2023-06-29T17:32:25.652685Z","shell.execute_reply.started":"2023-06-29T17:32:06.445025Z","shell.execute_reply":"2023-06-29T17:32:25.651404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"v_3 = model_2.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:25.654421Z","iopub.execute_input":"2023-06-29T17:32:25.654792Z","iopub.status.idle":"2023-06-29T17:32:46.329209Z","shell.execute_reply.started":"2023-06-29T17:32:25.654754Z","shell.execute_reply":"2023-06-29T17:32:46.327785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of prediction:\", v_1.shape)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:46.331075Z","iopub.execute_input":"2023-06-29T17:32:46.331621Z","iopub.status.idle":"2023-06-29T17:32:46.339064Z","shell.execute_reply.started":"2023-06-29T17:32:46.331570Z","shell.execute_reply":"2023-06-29T17:32:46.337579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testImgPredList_1 = [np.argmax(v_1[x]) for x in range(10)]\ntestImgPredList_1","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:46.340672Z","iopub.execute_input":"2023-06-29T17:32:46.341876Z","iopub.status.idle":"2023-06-29T17:32:46.355750Z","shell.execute_reply.started":"2023-06-29T17:32:46.341820Z","shell.execute_reply":"2023-06-29T17:32:46.354563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testImgPredList_2 = [np.argmax(v_2[x]) for x in range(10)]\ntestImgPredList_2","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:46.357432Z","iopub.execute_input":"2023-06-29T17:32:46.358447Z","iopub.status.idle":"2023-06-29T17:32:46.373236Z","shell.execute_reply.started":"2023-06-29T17:32:46.358404Z","shell.execute_reply":"2023-06-29T17:32:46.372132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testImgPredList_3 = [np.argmax(v_3[x]) for x in range(10)]\ntestImgPredList_3","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:46.374680Z","iopub.execute_input":"2023-06-29T17:32:46.375585Z","iopub.status.idle":"2023-06-29T17:32:46.388965Z","shell.execute_reply.started":"2023-06-29T17:32:46.375545Z","shell.execute_reply":"2023-06-29T17:32:46.387334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfor i in range (10):\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[i], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(testImgPredList_1[i])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:46.391017Z","iopub.execute_input":"2023-06-29T17:32:46.391433Z","iopub.status.idle":"2023-06-29T17:32:47.321461Z","shell.execute_reply.started":"2023-06-29T17:32:46.391395Z","shell.execute_reply":"2023-06-29T17:32:47.320121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range (10):\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[i], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(testImgPredList_2[i])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:47.323457Z","iopub.execute_input":"2023-06-29T17:32:47.324280Z","iopub.status.idle":"2023-06-29T17:32:48.248952Z","shell.execute_reply.started":"2023-06-29T17:32:47.324237Z","shell.execute_reply":"2023-06-29T17:32:48.247731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range (10):\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[i], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(testImgPredList_3[i])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:48.250691Z","iopub.execute_input":"2023-06-29T17:32:48.251070Z","iopub.status.idle":"2023-06-29T17:32:49.489433Z","shell.execute_reply.started":"2023-06-29T17:32:48.251031Z","shell.execute_reply":"2023-06-29T17:32:49.487981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Lets check the place where both the models have predicted differently\ndiff = [np.argmax(v_2[x]) - np.argmax(v_1[x]) for x in range(len(v_1))]\nindexList = [i for i,v in enumerate(diff) if v != 0]\nprint(\"Total different predictions: \", len(indexList))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:49.490813Z","iopub.execute_input":"2023-06-29T17:32:49.491190Z","iopub.status.idle":"2023-06-29T17:32:49.693692Z","shell.execute_reply.started":"2023-06-29T17:32:49.491154Z","shell.execute_reply":"2023-06-29T17:32:49.692307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets check which model is predicting better\nprint(\"First Model\")\nfor i in range (10):\n    actualIndex = indexList[i]\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[actualIndex], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(np.argmax(v_1[actualIndex]))\nplt.show()\n\nprint(\"--------------------------------\")\nprint(\"--------------------------------\")\nprint(\"Second Model\")\nfor i in range (10):\n    actualIndex = indexList[i]\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[actualIndex], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(np.argmax(v_2[actualIndex]))\nplt.show()\n\nprint(\"--------------------------------\")\nprint(\"--------------------------------\")\nprint(\"Third Model\")\nfor i in range (10):\n    actualIndex = indexList[i]\n    plt.subplot(2, 5, i+1)\n    img = np.reshape(X_test[actualIndex], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(np.argmax(v_3[actualIndex]))\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:49.695389Z","iopub.execute_input":"2023-06-29T17:32:49.695866Z","iopub.status.idle":"2023-06-29T17:32:52.467508Z","shell.execute_reply.started":"2023-06-29T17:32:49.695815Z","shell.execute_reply":"2023-06-29T17:32:52.466000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Going with first model\n#Creating the CSV to Submit\ny_pred = [np.argmax(x) for x in v_3]\n\nsubmissionDict = {'ImageId' : test_data.index + 1}\nsubmission = pd.DataFrame(submissionDict)\nsubmission['Label'] = y_pred","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:52.469563Z","iopub.execute_input":"2023-06-29T17:32:52.469961Z","iopub.status.idle":"2023-06-29T17:32:52.585367Z","shell.execute_reply.started":"2023-06-29T17:32:52.469924Z","shell.execute_reply":"2023-06-29T17:32:52.584298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:52.587670Z","iopub.execute_input":"2023-06-29T17:32:52.588651Z","iopub.status.idle":"2023-06-29T17:32:52.601562Z","shell.execute_reply.started":"2023-06-29T17:32:52.588588Z","shell.execute_reply":"2023-06-29T17:32:52.599781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/digit_submission_cnn.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-29T17:32:52.603720Z","iopub.execute_input":"2023-06-29T17:32:52.604133Z","iopub.status.idle":"2023-06-29T17:32:52.668396Z","shell.execute_reply.started":"2023-06-29T17:32:52.604093Z","shell.execute_reply":"2023-06-29T17:32:52.666780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}