{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-30T09:00:06.452240Z","iopub.execute_input":"2023-04-30T09:00:06.452777Z","iopub.status.idle":"2023-04-30T09:00:06.461818Z","shell.execute_reply.started":"2023-04-30T09:00:06.452722Z","shell.execute_reply":"2023-04-30T09:00:06.460293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:06.463808Z","iopub.execute_input":"2023-04-30T09:00:06.464178Z","iopub.status.idle":"2023-04-30T09:00:06.473758Z","shell.execute_reply.started":"2023-04-30T09:00:06.464137Z","shell.execute_reply":"2023-04-30T09:00:06.472791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename == 'train.csv':\n            train_data = pd.read_csv(os.path.join(dirname, filename))\n        elif filename == 'test.csv':\n            test_data = pd.read_csv(os.path.join(dirname, filename))\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:06.475951Z","iopub.execute_input":"2023-04-30T09:00:06.476419Z","iopub.status.idle":"2023-04-30T09:00:10.471653Z","shell.execute_reply.started":"2023-04-30T09:00:06.476370Z","shell.execute_reply":"2023-04-30T09:00:10.470789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_data.shape)\nprint(test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:10.473468Z","iopub.execute_input":"2023-04-30T09:00:10.473969Z","iopub.status.idle":"2023-04-30T09:00:10.478432Z","shell.execute_reply.started":"2023-04-30T09:00:10.473936Z","shell.execute_reply":"2023-04-30T09:00:10.477674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:10.480487Z","iopub.execute_input":"2023-04-30T09:00:10.481592Z","iopub.status.idle":"2023-04-30T09:00:10.503419Z","shell.execute_reply.started":"2023-04-30T09:00:10.481541Z","shell.execute_reply":"2023-04-30T09:00:10.502300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test data set has 784 columns, each column represent value of one pixel. Meaning the picture is 28X28X1\n\nTrain data set has one more column (feature) and that is target variable","metadata":{}},{"cell_type":"markdown","source":"Currently each pixel value for training and test dataset is stored in saperate column\nWe will need to reshape the data -> Creating N*28*28*1 numpy ND array","metadata":{}},{"cell_type":"code","source":"# Split the data into features and target\nX = train_data.drop('label', axis=1)  # Features\ny = train_data['label']  # Target\nX=X/255.0\nX=X.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:10.505425Z","iopub.execute_input":"2023-04-30T09:00:10.506289Z","iopub.status.idle":"2023-04-30T09:00:10.668701Z","shell.execute_reply.started":"2023-04-30T09:00:10.506220Z","shell.execute_reply":"2023-04-30T09:00:10.667615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgDataGenerator = tf.keras.preprocessing.image.ImageDataGenerator(\n    rescale = 1/255,\n    rotation_range = 20, \n    width_shift_range=0.02, height_shift_range=0.02,\n    shear_range=0.3,\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:10.669859Z","iopub.execute_input":"2023-04-30T09:00:10.670179Z","iopub.status.idle":"2023-04-30T09:00:10.675305Z","shell.execute_reply.started":"2023-04-30T09:00:10.670150Z","shell.execute_reply":"2023-04-30T09:00:10.674203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_lst = []\ny_lst = []\n\nfor index, x in enumerate(X):\n    X_lst.append(x)\n    y_lst.append(y[index])\n    for i in range(10):\n        dataPoint = imgDataGenerator.flow(np.reshape(x, (1, 28, 28, 1))).next()\n        X_lst.append(np.reshape(dataPoint, (28, 28, 1)))\n        y_lst.append(y[index])","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:10.677009Z","iopub.execute_input":"2023-04-30T09:00:10.677448Z","iopub.status.idle":"2023-04-30T09:00:27.814838Z","shell.execute_reply.started":"2023-04-30T09:00:10.677346Z","shell.execute_reply":"2023-04-30T09:00:27.813696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.array(X_lst)\ny = np.array(y_lst)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:27.816583Z","iopub.execute_input":"2023-04-30T09:00:27.817158Z","iopub.status.idle":"2023-04-30T09:00:28.088624Z","shell.execute_reply.started":"2023-04-30T09:00:27.817121Z","shell.execute_reply":"2023-04-30T09:00:28.087577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X,y, test_size=0.2, random_state=42,shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:28.089789Z","iopub.execute_input":"2023-04-30T09:00:28.090402Z","iopub.status.idle":"2023-04-30T09:00:28.327219Z","shell.execute_reply.started":"2023-04-30T09:00:28.090366Z","shell.execute_reply":"2023-04-30T09:00:28.325785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets now visualize the training image","metadata":{}},{"cell_type":"code","source":"img = np.reshape(X[12], (28, 28))\nplt.imshow(img, cmap='gray')\nplt.show()\nimg = np.reshape(X[13], (28, 28))\nplt.imshow(img, cmap='gray')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:28.328908Z","iopub.execute_input":"2023-04-30T09:00:28.330055Z","iopub.status.idle":"2023-04-30T09:00:28.759264Z","shell.execute_reply.started":"2023-04-30T09:00:28.329985Z","shell.execute_reply":"2023-04-30T09:00:28.758248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now that we have the training data in ND array form, we can create a simple CNN model and train it on given data","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))\nmodel.add(Conv2D(32, kernel_size=(3, 3),activation='relu'))\nmodel.add(MaxPool2D((2, 2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))\nmodel.add(Conv2D(64, kernel_size=(3, 3),activation='relu'))\nmodel.add(MaxPool2D((2, 2)))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.25))\n\nmodel.add(Dense(128, activation = \"relu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(64, activation = \"relu\"))\nmodel.add(BatchNormalization())\nmodel.add(Dropout(0.5))\n\nmodel.add(Dense(10, activation = \"softmax\"))\n\nmodel.compile(loss=keras.losses.sparse_categorical_crossentropy,\n              optimizer=keras.optimizers.RMSprop(),\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:28.763488Z","iopub.execute_input":"2023-04-30T09:00:28.764399Z","iopub.status.idle":"2023-04-30T09:00:29.327967Z","shell.execute_reply.started":"2023-04-30T09:00:28.764361Z","shell.execute_reply":"2023-04-30T09:00:29.326836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(X_train, y_train, epochs=15,batch_size=256, validation_data=(X_val, y_val))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:00:29.329335Z","iopub.execute_input":"2023-04-30T09:00:29.329657Z","iopub.status.idle":"2023-04-30T09:02:01.102073Z","shell.execute_reply.started":"2023-04-30T09:00:29.329625Z","shell.execute_reply":"2023-04-30T09:02:01.100885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets Test the classification of this model on Test Data Set","metadata":{"execution":{"iopub.status.busy":"2023-04-22T18:00:41.23797Z","iopub.execute_input":"2023-04-22T18:00:41.238413Z","iopub.status.idle":"2023-04-22T18:00:41.246948Z","shell.execute_reply.started":"2023-04-22T18:00:41.238375Z","shell.execute_reply":"2023-04-22T18:00:41.245436Z"}}},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:02:01.103980Z","iopub.execute_input":"2023-04-30T09:02:01.104397Z","iopub.status.idle":"2023-04-30T09:02:01.109248Z","shell.execute_reply.started":"2023-04-30T09:02:01.104364Z","shell.execute_reply":"2023-04-30T09:02:01.108213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the data into features and target\nX_test = test_data\nX_test=X_test/255.0\nX_test=X_test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:02:01.110977Z","iopub.execute_input":"2023-04-30T09:02:01.111344Z","iopub.status.idle":"2023-04-30T09:02:01.169363Z","shell.execute_reply.started":"2023-04-30T09:02:01.111312Z","shell.execute_reply":"2023-04-30T09:02:01.168276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"we can see that \"0\" is being missclassified as \"9\"","metadata":{}},{"cell_type":"code","source":"#List of indices where model has classified the digit as \"9\"\nv_train = model.predict(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:07:54.564153Z","iopub.execute_input":"2023-04-30T09:07:54.564559Z","iopub.status.idle":"2023-04-30T09:08:21.575921Z","shell.execute_reply.started":"2023-04-30T09:07:54.564523Z","shell.execute_reply":"2023-04-30T09:08:21.574728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#indexList = [i for i,v in enumerate(v_train) if (np.argmax(v) == 9 and y_train_reset.iloc[i].label != 9)]\nindexList = [i for i,v in enumerate(v_train) if (np.argmax(v) == 8 and y_train[i] != 8)]\nprint(\"Total different predictions: \", len(indexList))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:11:57.319646Z","iopub.execute_input":"2023-04-30T09:11:57.320112Z","iopub.status.idle":"2023-04-30T09:11:57.499903Z","shell.execute_reply.started":"2023-04-30T09:11:57.320074Z","shell.execute_reply":"2023-04-30T09:11:57.498540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limit = len(indexList) if len(indexList) < 5 else 5\nfig, ax = plt.subplots(nrows=1, ncols=5, figsize=(15,5))\nfor i in range (limit):\n    actualIndex = indexList[i]\n    img = np.reshape(X_train[actualIndex], (28, 28))\n    ax[i].imshow(img, cmap='gray')\n    ax[i].title.set_text(\"Classified as:\" + str(np.argmax(v_train[actualIndex])))\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:15:26.670294Z","iopub.execute_input":"2023-04-30T09:15:26.671438Z","iopub.status.idle":"2023-04-30T09:15:27.233502Z","shell.execute_reply.started":"2023-04-30T09:15:26.671390Z","shell.execute_reply":"2023-04-30T09:15:27.232257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let check where model has classified \"0\"\n","metadata":{}},{"cell_type":"code","source":"#List of indices where model has classified the digit as \"0\"\n\n#indexList = [i for i,v in enumerate(v_train) if (np.argmax(v) == 0 and y_train_reset.iloc[i].label != 0)]\nindexList = [i for i,v in enumerate(v_train) if (np.argmax(v) == 0 and y_train[i] != 0)]\nprint(\"Total different predictions: \", len(indexList))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:16:46.456594Z","iopub.execute_input":"2023-04-30T09:16:46.456994Z","iopub.status.idle":"2023-04-30T09:16:46.613248Z","shell.execute_reply.started":"2023-04-30T09:16:46.456960Z","shell.execute_reply":"2023-04-30T09:16:46.611636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limit = len(indexList) if len(indexList) < 5 else 5\nfig, ax = plt.subplots(nrows=1, ncols=5, figsize=(15,5))\nfor i in range (limit):\n    actualIndex = indexList[i]\n    img = np.reshape(X_train[actualIndex], (28, 28))\n    ax[i].imshow(img, cmap='gray')\n    ax[i].title.set_text(\"Classified as:\" + str(np.argmax(v_train[actualIndex])))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:16:48.455447Z","iopub.execute_input":"2023-04-30T09:16:48.455845Z","iopub.status.idle":"2023-04-30T09:16:49.004313Z","shell.execute_reply.started":"2023-04-30T09:16:48.455810Z","shell.execute_reply":"2023-04-30T09:16:49.003104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets also check where model has not classified the digit as \"9\" but it was actually 9.\nand same for 0","metadata":{}},{"cell_type":"code","source":"#indexList = [i for i,v in enumerate(v_train) if (np.argmax(v) != 9 and y_train_reset.iloc[i].label == 9)]\nindexList = [i for i,v in enumerate(v_train) if (np.argmax(v) != 9 and y_train[i] == 9)]\nprint(\"Total different predictions: \", len(indexList))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:16:56.090087Z","iopub.execute_input":"2023-04-30T09:16:56.090947Z","iopub.status.idle":"2023-04-30T09:16:56.258342Z","shell.execute_reply.started":"2023-04-30T09:16:56.090901Z","shell.execute_reply":"2023-04-30T09:16:56.257163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limit = len(indexList) if len(indexList) < 5 else 5\nfig, ax = plt.subplots(nrows=1, ncols=5, figsize=(15,5))\nfor i in range (limit):\n    actualIndex = indexList[i]\n    img = np.reshape(X_train[actualIndex], (28, 28))\n    ax[i].imshow(img, cmap='gray')\n    ax[i].title.set_text(\"Classified as:\" + str(np.argmax(v_train[actualIndex])))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:16:58.641516Z","iopub.execute_input":"2023-04-30T09:16:58.641938Z","iopub.status.idle":"2023-04-30T09:16:59.176184Z","shell.execute_reply.started":"2023-04-30T09:16:58.641900Z","shell.execute_reply":"2023-04-30T09:16:59.174852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#indexList = [i for i,v in enumerate(v_train) if (np.argmax(v) != 0 and y_train_reset.iloc[i].label == 0)]\nindexList = [i for i,v in enumerate(v_train) if (np.argmax(v) != 0 and y_train[i] == 0)]\nprint(\"Total different predictions: \", len(indexList))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:17:42.401077Z","iopub.execute_input":"2023-04-30T09:17:42.401521Z","iopub.status.idle":"2023-04-30T09:17:42.567605Z","shell.execute_reply.started":"2023-04-30T09:17:42.401485Z","shell.execute_reply":"2023-04-30T09:17:42.566308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"limit = len(indexList) if len(indexList) < 5 else 5\nfig, ax = plt.subplots(nrows=1, ncols=5, figsize=(15,5))\nfor i in range (limit):\n    actualIndex = indexList[i]\n    img = np.reshape(X_train[actualIndex], (28, 28))\n    ax[i].imshow(img, cmap='gray')\n    ax[i].title.set_text(\"Classified as:\" + str(np.argmax(v_train[actualIndex])))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:17:43.871915Z","iopub.execute_input":"2023-04-30T09:17:43.872349Z","iopub.status.idle":"2023-04-30T09:17:44.431081Z","shell.execute_reply.started":"2023-04-30T09:17:43.872310Z","shell.execute_reply":"2023-04-30T09:17:44.429572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets also count the maximum misclassification was done for which digit","metadata":{}},{"cell_type":"code","source":"#indexValTupleList = [(i,y_train_reset.iloc[i].label) for i,v in enumerate(v_train) if (np.argmax(v) != y_train_reset.iloc[i].label)]\nindexValTupleList = [(i,y_train[i]) for i,v in enumerate(v_train) if (np.argmax(v) != y_train[i])]\nprint(\"Total different predictions: \", len(indexValTupleList))","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:18:01.247384Z","iopub.execute_input":"2023-04-30T09:18:01.248505Z","iopub.status.idle":"2023-04-30T09:18:01.413378Z","shell.execute_reply.started":"2023-04-30T09:18:01.248446Z","shell.execute_reply":"2023-04-30T09:18:01.412113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(indexValTupleList, columns=['Index', 'Digit'])\ndf.Digit.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-04-30T09:18:05.135889Z","iopub.execute_input":"2023-04-30T09:18:05.137399Z","iopub.status.idle":"2023-04-30T09:18:05.182629Z","shell.execute_reply.started":"2023-04-30T09:18:05.137341Z","shell.execute_reply":"2023-04-30T09:18:05.181458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The Errors in the classification seems to be because of the extremly poor hand writing\n\nThus, the model is good to use on test data set and generate the submission","metadata":{}},{"cell_type":"code","source":"v_1 = model.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testImgPredList_1 = [np.argmax(v_1[x]) for x in range(50)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Shape of prediction:\", v_1.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nplt.figure(figsize=(15,15))\n\nfor i in range (50):\n    plt.subplot(5, 10, i+1)\n    img = np.reshape(X_test[i], (28, 28))\n    plt.imshow(img, cmap='gray')\n    plt.title(\"Classified as: \" + str(testImgPredList_1[i]))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Going with first model\n#Creating the CSV to Submit\ny_pred = [np.argmax(x) for x in v_1]\n\nsubmissionDict = {'ImageId' : test_data.index + 1}\nsubmission = pd.DataFrame(submissionDict)\nsubmission['Label'] = y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('/kaggle/working/digit_submission_deepNet.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}