{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Initialization - Read Libraries and Training Data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom PIL import Image\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:32.326187Z","iopub.execute_input":"2022-04-09T01:39:32.326984Z","iopub.status.idle":"2022-04-09T01:39:32.352893Z","shell.execute_reply.started":"2022-04-09T01:39:32.326876Z","shell.execute_reply":"2022-04-09T01:39:32.352237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/digit-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:32.354321Z","iopub.execute_input":"2022-04-09T01:39:32.354654Z","iopub.status.idle":"2022-04-09T01:39:36.017928Z","shell.execute_reply.started":"2022-04-09T01:39:32.354626Z","shell.execute_reply":"2022-04-09T01:39:36.017225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## EDA","metadata":{}},{"cell_type":"code","source":"def understand_variables(dataset):\n    #print(\"Type = \" +str(type(dataset))+\"\\n\")\n    print(\"Shape = \"+str(dataset.shape)+\"\\n\")\n    #print(\"Head : \\n\\n\"+str(dataset.head())+\"\\n\\n\")\n    #print(\"Columns:\\n\"+str(dataset.columns)+\"\\n\\n\")\n    print(\"No.of unique values :\\n\\n\"+str(dataset.nunique(axis=0).sort_values(ascending=False))+\"\\n\\n\")\n    print(\"Description :\\n\\n\"+str(dataset.describe())+\"\\n\\n\")\n    \n    #print(dataset.describe(exclude=[np.number]))\n    #Since no categorical variables, no need to have the above line\n    \n    print(\"Null count :\\n\\n\"+str(dataset.columns[train_data.isnull().sum()!=0])+\"\\n\\n\")\n    \n    Columns_with_1_unique_value = train_data.nunique(axis=0).sort_values(ascending=False)\n    \n    print(\"No.of columns that could be excluded (1 unique value): \"+str(len(Columns_with_1_unique_value[Columns_with_1_unique_value==1])))\n","metadata":{"_kg_hide-input":false,"_kg_hide-output":false,"execution":{"iopub.status.busy":"2022-04-09T01:39:36.019434Z","iopub.execute_input":"2022-04-09T01:39:36.019877Z","iopub.status.idle":"2022-04-09T01:39:36.028422Z","shell.execute_reply.started":"2022-04-09T01:39:36.019836Z","shell.execute_reply":"2022-04-09T01:39:36.027392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"understand_variables(train_data)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:36.031286Z","iopub.execute_input":"2022-04-09T01:39:36.031504Z","iopub.status.idle":"2022-04-09T01:39:38.810876Z","shell.execute_reply.started":"2022-04-09T01:39:36.031477Z","shell.execute_reply":"2022-04-09T01:39:38.810020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Inferences**\n\nLarge Data set with 42000 images, each with a dimension of root(784) = 28 * 28\n\nPixel columns have values in range of 0-255\n\nNo columns have null values (as expected since it's image data)","metadata":{}},{"cell_type":"markdown","source":"### Image Visual EDA ","metadata":{}},{"cell_type":"code","source":"print(\"Label : \" + str(train_data.iloc[512][0]))","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:38.812160Z","iopub.execute_input":"2022-04-09T01:39:38.812404Z","iopub.status.idle":"2022-04-09T01:39:38.818363Z","shell.execute_reply.started":"2022-04-09T01:39:38.812374Z","shell.execute_reply":"2022-04-09T01:39:38.817490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_1D = train_data.iloc[512][1:]\nprint(\"Image Data\\n\")\nimage_1D","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:38.819781Z","iopub.execute_input":"2022-04-09T01:39:38.820014Z","iopub.status.idle":"2022-04-09T01:39:38.835888Z","shell.execute_reply.started":"2022-04-09T01:39:38.819985Z","shell.execute_reply":"2022-04-09T01:39:38.834958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"10 digits at Random\\n\")\n\nfor j in range(230,240):\n    \n    image_1D = train_data.iloc[j][1:]\n\n    side_len = int(np.sqrt(len(image_1D)))\n    image_2D = np.zeros(shape=(side_len,side_len))\n\n    for i in range(0,len(image_1D)):\n\n        row = int(i/side_len)\n        col = int(i%side_len)\n\n        #print(row,col)\n        \n        #if image_1D[i]!=0:\n        #    image_1D[i] = 1\n\n        image_2D[row,col] = image_1D[i]\n\n    print(train_data.iloc[j][0])   \n    #print(\"----------------\")\n    plt.figure()        \n    plt.imshow(image_2D)\n    #print(\"----------------\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:38.836926Z","iopub.execute_input":"2022-04-09T01:39:38.837150Z","iopub.status.idle":"2022-04-09T01:39:40.662819Z","shell.execute_reply.started":"2022-04-09T01:39:38.837114Z","shell.execute_reply":"2022-04-09T01:39:40.662028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The image makes sense with the corresponding label number. However, we don't need the range of colors displayed here, only grayscale would suffice. \n\nHence, next step could be to **normalize the Pixel range from 0-255 to 0-1**","metadata":{}},{"cell_type":"code","source":"train_data.iloc[:,1:] = train_data.iloc[:,1:]/255","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:39:40.664794Z","iopub.execute_input":"2022-04-09T01:39:40.665032Z","iopub.status.idle":"2022-04-09T01:40:12.499604Z","shell.execute_reply.started":"2022-04-09T01:39:40.664989Z","shell.execute_reply":"2022-04-09T01:40:12.498718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:12.501120Z","iopub.execute_input":"2022-04-09T01:40:12.501530Z","iopub.status.idle":"2022-04-09T01:40:12.548348Z","shell.execute_reply.started":"2022-04-09T01:40:12.501486Z","shell.execute_reply":"2022-04-09T01:40:12.547786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"10 digits at Random\\n\")\n\nfor j in range(230,240):\n    \n    image_1D = train_data.iloc[j][1:]\n\n    side_len = int(np.sqrt(len(image_1D)))\n    image_2D = np.zeros(shape=(side_len,side_len))\n\n    for i in range(0,len(image_1D)):\n\n        row = int(i/side_len)\n        col = int(i%side_len)\n\n        #print(row,col)\n        \n        #if image_1D[i]!=0:\n        #    image_1D[i] = 1\n\n        image_2D[row,col] = image_1D[i]\n\n    print(train_data.iloc[j][0])   \n    #print(\"----------------\")\n    plt.figure()        \n    plt.imshow(image_2D)\n    #print(\"----------------\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:12.551670Z","iopub.execute_input":"2022-04-09T01:40:12.552262Z","iopub.status.idle":"2022-04-09T01:40:14.591946Z","shell.execute_reply.started":"2022-04-09T01:40:12.552221Z","shell.execute_reply":"2022-04-09T01:40:14.591103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As this is a classification problem (although targets are digits), it would be better to convert the target to **String**.","metadata":{}},{"cell_type":"code","source":"train_data.label = train_data.label.astype(float).astype(int).astype(str)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:14.593251Z","iopub.execute_input":"2022-04-09T01:40:14.593443Z","iopub.status.idle":"2022-04-09T01:40:14.651750Z","shell.execute_reply.started":"2022-04-09T01:40:14.593418Z","shell.execute_reply":"2022-04-09T01:40:14.651133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Example of Convolution, Detection (ReLu), Max Pooling on this data ","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nkernel = tf.constant([\n    [-1, -1, -1],\n    [-1,  8, -1],\n    [-1, -1, -1],\n])\n\nkernel = tf.reshape(kernel, [*kernel.shape, 1, 1])\nkernel = tf.cast(kernel, dtype=tf.float32)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:14.652853Z","iopub.execute_input":"2022-04-09T01:40:14.653126Z","iopub.status.idle":"2022-04-09T01:40:20.011864Z","shell.execute_reply.started":"2022-04-09T01:40:14.653058Z","shell.execute_reply":"2022-04-09T01:40:20.010984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reformat for batch compatibility.\n#image = tf.image.convert_image_dtype(image_2D, dtype=tf.float32)\nimage = tf.reshape(image_2D, [1,*image_2D.shape, 1])\nimage = tf.cast(image, dtype=tf.float32)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:20.013641Z","iopub.execute_input":"2022-04-09T01:40:20.014288Z","iopub.status.idle":"2022-04-09T01:40:20.022672Z","shell.execute_reply.started":"2022-04-09T01:40:20.014235Z","shell.execute_reply":"2022-04-09T01:40:20.022047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:20.024157Z","iopub.execute_input":"2022-04-09T01:40:20.025110Z","iopub.status.idle":"2022-04-09T01:40:20.032953Z","shell.execute_reply.started":"2022-04-09T01:40:20.025045Z","shell.execute_reply":"2022-04-09T01:40:20.031951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernel.shape","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:20.035046Z","iopub.execute_input":"2022-04-09T01:40:20.035468Z","iopub.status.idle":"2022-04-09T01:40:20.046444Z","shell.execute_reply.started":"2022-04-09T01:40:20.035438Z","shell.execute_reply":"2022-04-09T01:40:20.045477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convolution\n\nplt.figure(figsize=(6, 6))\nplt.imshow(tf.squeeze(image))\nplt.axis('off')\nplt.show();\n\nimage_filter = tf.nn.conv2d(\n    image,\n    kernel,\n    strides=1,\n    padding='SAME'\n)\n\nplt.figure(figsize=(6, 6))\nplt.imshow(tf.squeeze(image_filter))\nplt.axis('off')\nplt.show();\n\n# Detection\n\nimage_detect = tf.nn.relu(image_filter)\n\nplt.figure(figsize=(6, 6))\nplt.imshow(tf.squeeze(image_detect))\nplt.axis('off')\nplt.show();\n\n# Max Pooling\n\nimage_condense = tf.nn.pool(\n    input=image_detect, # image in the Detect step above\n    window_shape=(2, 2),\n    pooling_type='MAX',\n    # we'll see what these do in the next lesson!\n    strides=(2, 2),\n    padding='SAME',\n)\n\nplt.figure(figsize=(6, 6))\nplt.imshow(tf.squeeze(image_condense))\nplt.axis('off')\nplt.show();\n\n#image = image_condense","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:20.047808Z","iopub.execute_input":"2022-04-09T01:40:20.048314Z","iopub.status.idle":"2022-04-09T01:40:20.551415Z","shell.execute_reply.started":"2022-04-09T01:40:20.048268Z","shell.execute_reply":"2022-04-09T01:40:20.550307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let us take 2 images with same target of 2, and find out how they look after above processing with 2 layers","metadata":{}},{"cell_type":"code","source":"print(\"2 digits with label as \\'2\\''\\n\")\n\nIndex_of_2 = list(train_data[train_data.label=='2'].index)\n\nfor j in Index_of_2[:2]:\n    \n    image_1D = train_data.iloc[j][1:]\n\n    side_len = int(np.sqrt(len(image_1D)))\n    image_2D = np.zeros(shape=(side_len,side_len))\n\n    for i in range(0,len(image_1D)):\n\n        row = int(i/side_len)\n        col = int(i%side_len)\n\n        #print(row,col)\n        \n        #if image_1D[i]!=0:\n        #    image_1D[i] = 1\n\n        image_2D[row,col] = image_1D[i]\n\n    print(train_data.iloc[j][0])   \n    print(\"----------------\")\n    #plt.figure()        \n    #plt.imshow(image_2D)\n    #print(\"----------------\")\n    \n    # Reformat for batch compatibility.\n    #image = tf.image.convert_image_dtype(image_2D, dtype=tf.float32)\n    image = tf.reshape(image_2D, [1,*image_2D.shape, 1])\n    image = tf.cast(image, dtype=tf.float32)\n    \n    plt.figure(figsize=(6, 6))\n    plt.imshow(tf.squeeze(image))\n    plt.axis('off')\n    plt.show();\n\n    for i in range(0,2):\n    \n        image_filter = tf.nn.conv2d(\n            image,\n            kernel,\n            strides=1,\n            padding='SAME'\n        )\n\n        plt.figure(figsize=(6, 6))\n        plt.imshow(tf.squeeze(image_filter))\n        plt.axis('off')\n        plt.show();\n\n        image_detect = tf.nn.relu(image_filter)\n\n        plt.figure(figsize=(6, 6))\n        plt.imshow(tf.squeeze(image_detect))\n        plt.axis('off')\n        plt.show();\n\n        image_condense = tf.nn.pool(\n            input=image_detect, # image in the Detect step above\n            window_shape=(2, 2),\n            pooling_type='MAX',\n            # we'll see what these do in the next lesson!\n            strides=(2, 2),\n            padding='SAME',\n        )\n\n        plt.figure(figsize=(6, 6))\n        plt.imshow(tf.squeeze(image_condense))\n        plt.axis('off')\n        plt.show();\n\n        image = image_condense","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:20.554558Z","iopub.execute_input":"2022-04-09T01:40:20.554896Z","iopub.status.idle":"2022-04-09T01:40:21.835814Z","shell.execute_reply.started":"2022-04-09T01:40:20.554855Z","shell.execute_reply":"2022-04-09T01:40:21.834838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling - CNN","metadata":{}},{"cell_type":"markdown","source":"## Reshaping the data from 1D to 2D","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest_data = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:21.837265Z","iopub.execute_input":"2022-04-09T01:40:21.837553Z","iopub.status.idle":"2022-04-09T01:40:26.728414Z","shell.execute_reply.started":"2022-04-09T01:40:21.837501Z","shell.execute_reply":"2022-04-09T01:40:26.727773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.preprocessing import image_dataset_from_directory\n\ntrain_data_X = train_data.values[:,1:].reshape(-1,28, 28,1).astype('float32')/255\ntrain_data_Y = train_data.values[:,0].astype('str')\ntrain_data_Y = tf.one_hot(train_data_Y,10)\n\ntest_data_X = test_data.values.reshape(-1,28, 28,1).astype('float32')/255","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:26.729599Z","iopub.execute_input":"2022-04-09T01:40:26.733154Z","iopub.status.idle":"2022-04-09T01:40:28.206813Z","shell.execute_reply.started":"2022-04-09T01:40:26.733115Z","shell.execute_reply":"2022-04-09T01:40:28.205943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN Implementation - Custom Covnet","metadata":{}},{"cell_type":"markdown","source":"2-> 3 CNN layers \n\n95 -> 99 Accuracy on Kaggle Submission","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras import layers\n\nmodel = keras.Sequential([\n    \n    layers.InputLayer(input_shape=(28, 28,1)),\n\n    # First Convolutional Block\n    layers.Conv2D(filters=32, kernel_size=3, activation=\"relu\", padding='same'),                  \n    layers.MaxPool2D(),    \n    layers.Dropout(rate=0.2),\n    layers.BatchNormalization(),\n\n\n    # Second Convolutional Block\n    layers.Conv2D(filters=64, kernel_size=3, activation=\"relu\", padding='same'),\n    layers.MaxPool2D(),\n    layers.Dropout(rate=0.2),\n    layers.BatchNormalization(),\n\n    # Third Convolutional Block\n    layers.Conv2D(filters=128, kernel_size=3, activation=\"relu\", padding='same'),\n    layers.MaxPool2D(),\n    layers.Dropout(rate=0.2),\n    layers.BatchNormalization(),\n\n    \n    # Fourth Convolutional Block\n    layers.Conv2D(filters=256, kernel_size=3, activation=\"relu\", padding='same'),\n    layers.MaxPool2D(),\n    layers.Dropout(rate=0.2),\n    layers.BatchNormalization(),\n    \n    # Classifier Head\n    layers.Flatten(),\n\n    layers.Dense(units=256, activation=\"relu\"),\n    layers.Dropout(rate=0.2),\n    layers.BatchNormalization(),\n    \n    layers.Dense(units=10, activation=\"sigmoid\"),\n])\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:28.208219Z","iopub.execute_input":"2022-04-09T01:40:28.209110Z","iopub.status.idle":"2022-04-09T01:40:28.424238Z","shell.execute_reply.started":"2022-04-09T01:40:28.209036Z","shell.execute_reply":"2022-04-09T01:40:28.423318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='Adam',\n    loss='categorical_crossentropy',\n    metrics=['accuracy']\n)\n\nfrom tensorflow.keras.callbacks import EarlyStopping\n\n\n\nhistory = model.fit(\n    train_data_X,\n    train_data_Y,\n    validation_data=None,\n    epochs=20,\n)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:40:28.427601Z","iopub.execute_input":"2022-04-09T01:40:28.427844Z","iopub.status.idle":"2022-04-09T01:55:49.655163Z","shell.execute_reply.started":"2022-04-09T01:40:28.427814Z","shell.execute_reply":"2022-04-09T01:55:49.654271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_Y = model.predict(test_data_X)","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:55:49.656681Z","iopub.execute_input":"2022-04-09T01:55:49.656894Z","iopub.status.idle":"2022-04-09T01:55:58.298863Z","shell.execute_reply.started":"2022-04-09T01:55:49.656868Z","shell.execute_reply":"2022-04-09T01:55:58.298141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_Y","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:55:58.299942Z","iopub.execute_input":"2022-04-09T01:55:58.300543Z","iopub.status.idle":"2022-04-09T01:55:58.306811Z","shell.execute_reply.started":"2022-04-09T01:55:58.300509Z","shell.execute_reply":"2022-04-09T01:55:58.306275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data_Y = [np.argmax(x) for x in test_data_Y]","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:55:58.307918Z","iopub.execute_input":"2022-04-09T01:55:58.308305Z","iopub.status.idle":"2022-04-09T01:55:58.402915Z","shell.execute_reply.started":"2022-04-09T01:55:58.308276Z","shell.execute_reply":"2022-04-09T01:55:58.402082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_digits = pd.DataFrame(test_data_Y)\noutput_digits.columns = ['Label']\noutput_digits.index.names = ['ImageId']\noutput_digits.index = output_digits.index+1\nprint(output_digits)\noutput_digits.to_csv(\"Digit Recognizer Output.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-04-09T01:55:58.404279Z","iopub.execute_input":"2022-04-09T01:55:58.404725Z","iopub.status.idle":"2022-04-09T01:55:58.495437Z","shell.execute_reply.started":"2022-04-09T01:55:58.404693Z","shell.execute_reply":"2022-04-09T01:55:58.494700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To submit the output, download the file from the output data on the right, then upload it to competition","metadata":{}},{"cell_type":"markdown","source":"# Kaggle submission yields 99.3% accuracy","metadata":{}}]}