{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importing Required Libraries\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Conv2D, MaxPooling2D, Flatten, Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.regularizers import l2\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:01:51.827523Z","iopub.execute_input":"2021-07-02T12:01:51.828031Z","iopub.status.idle":"2021-07-02T12:01:57.702667Z","shell.execute_reply.started":"2021-07-02T12:01:51.827952Z","shell.execute_reply":"2021-07-02T12:01:57.701559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading Data from provided .csv file\ndata = pd.read_csv('../input/digit-recognizer/train.csv')\nprint(data.info())\ndata.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:03:44.844997Z","iopub.execute_input":"2021-07-02T12:03:44.845298Z","iopub.status.idle":"2021-07-02T12:03:47.959646Z","shell.execute_reply.started":"2021-07-02T12:03:44.845267Z","shell.execute_reply":"2021-07-02T12:03:47.958956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieving labels from Data\nlabels = data['label']\n\n# Converting into One-Hot Vectors for Classification\nlabels = pd.get_dummies(labels)\nlabels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:03:53.748151Z","iopub.execute_input":"2021-07-02T12:03:53.748464Z","iopub.status.idle":"2021-07-02T12:03:53.758011Z","shell.execute_reply.started":"2021-07-02T12:03:53.748439Z","shell.execute_reply":"2021-07-02T12:03:53.757359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieving all pixel columns from Data\ndata = data.iloc[:,1:]\ndata.shape ","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:03:55.367272Z","iopub.execute_input":"2021-07-02T12:03:55.367744Z","iopub.status.idle":"2021-07-02T12:03:55.374402Z","shell.execute_reply.started":"2021-07-02T12:03:55.367711Z","shell.execute_reply":"2021-07-02T12:03:55.373429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking 40000 images to Train the model\ntrain_data = data[2000:]\ntrain_labels = labels[2000:]\n\ntrain_data.shape, train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:04:00.270153Z","iopub.execute_input":"2021-07-02T12:04:00.270767Z","iopub.status.idle":"2021-07-02T12:04:00.277312Z","shell.execute_reply.started":"2021-07-02T12:04:00.270717Z","shell.execute_reply":"2021-07-02T12:04:00.276598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking 2000 images for Cross Validation\ntest_data = data[:2000]\ntest_labels = labels[:2000]\n\ntest_data.shape, test_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:04:01.425387Z","iopub.execute_input":"2021-07-02T12:04:01.425743Z","iopub.status.idle":"2021-07-02T12:04:01.432052Z","shell.execute_reply.started":"2021-07-02T12:04:01.425713Z","shell.execute_reply":"2021-07-02T12:04:01.431150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standard loc and scale values for kernel initializer \ndef initialize_weights(shape, dtype=None):\n    \n    return np.random.normal(loc = 0.0, scale = 1e-2, size = shape)\n\n# Standard loc and scale values for bias initializer\ndef initialize_bias(shape, dtype=None):\n    \n    return np.random.normal(loc = 0.5, scale = 1e-2, size = shape)\n\ndef DeepLearningModel(input_shape):\n    model = Sequential()\n    \n    model.add(Input(input_shape))\n    \n    model.add(Dense(64, activation='relu', kernel_initializer=initialize_weights, bias_initializer=initialize_bias, kernel_regularizer=l2(3e-3)))\n    \n    model.add(Dense(128, activation='relu', kernel_initializer=initialize_weights, bias_initializer=initialize_bias, kernel_regularizer=l2(3e-3)))\n\n    model.add(Dense(256, activation='relu', kernel_initializer=initialize_weights, bias_initializer=initialize_bias, kernel_regularizer=l2(3e-3)))\n                \n    model.add(Dense(10, activation='softmax', kernel_initializer=initialize_weights, bias_initializer=initialize_bias, kernel_regularizer=l2(3e-3)))    \n    \n    return model\n\n\nmodel = DeepLearningModel((784))\nmodel.summary()","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-07-02T12:04:03.508251Z","iopub.execute_input":"2021-07-02T12:04:03.508604Z","iopub.status.idle":"2021-07-02T12:04:03.589319Z","shell.execute_reply.started":"2021-07-02T12:04:03.508566Z","shell.execute_reply":"2021-07-02T12:04:03.588703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters\nlr = 0.0003\nepochs = 50\nbatch_size = 128\n\noptimizer = Adam(lr)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\nhistory = model.fit(train_data, train_labels, epochs=epochs, batch_size=batch_size)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-07-02T12:04:08.071334Z","iopub.execute_input":"2021-07-02T12:04:08.071878Z","iopub.status.idle":"2021-07-02T12:05:00.707265Z","shell.execute_reply.started":"2021-07-02T12:04:08.071844Z","shell.execute_reply":"2021-07-02T12:05:00.706410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluating model using Validation Data\nmodel.evaluate(test_data, test_labels)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:05:15.792616Z","iopub.execute_input":"2021-07-02T12:05:15.793019Z","iopub.status.idle":"2021-07-02T12:05:16.073532Z","shell.execute_reply.started":"2021-07-02T12:05:15.792985Z","shell.execute_reply":"2021-07-02T12:05:16.072759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading Test Dataset\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\ntest.info()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:05:28.256280Z","iopub.execute_input":"2021-07-02T12:05:28.256642Z","iopub.status.idle":"2021-07-02T12:05:30.123947Z","shell.execute_reply.started":"2021-07-02T12:05:28.256611Z","shell.execute_reply":"2021-07-02T12:05:30.122800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making Predictions on existing model.\nprediction = model.predict(test)\n\n# Rounding the prediction and converting float64 output to int64 output.\nprediction = np.array(np.round(prediction), dtype='int64')\n\n# Converting one-hot encoding into label encoding\nprediction = (np.argmax(prediction, axis=1)).reshape(-1, 1)\n\n# Creating a DataFrame similar to Example Submission\nout = [{'ImageId': i+1, 'Label': prediction[i][0]} for i in range(len(prediction))]\n\n# Creating a .csv file from the DataFrame\npd.DataFrame(out).to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:06:36.509335Z","iopub.execute_input":"2021-07-02T12:06:36.509725Z","iopub.status.idle":"2021-07-02T12:06:37.711043Z","shell.execute_reply.started":"2021-07-02T12:06:36.509684Z","shell.execute_reply":"2021-07-02T12:06:37.709882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Achieved 97% accuracy with Deep Learning Classifier\n\nNow let's see how a Convolutional Neural Network will perform on the Dataset.","metadata":{}},{"cell_type":"code","source":"imgdata = np.reshape(np.array(data),(data.shape[0],28,28,1))\nimglabels = labels\n\nimgdata.shape, imglabels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:06:54.429857Z","iopub.execute_input":"2021-07-02T12:06:54.430212Z","iopub.status.idle":"2021-07-02T12:06:54.541312Z","shell.execute_reply.started":"2021-07-02T12:06:54.430184Z","shell.execute_reply":"2021-07-02T12:06:54.540464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking 40000 images to Train the model\nimg_train_data = imgdata[2000:]\nimg_train_labels = imglabels[2000:]\n\nimg_train_data.shape, img_train_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:07:01.254971Z","iopub.execute_input":"2021-07-02T12:07:01.255343Z","iopub.status.idle":"2021-07-02T12:07:01.261938Z","shell.execute_reply.started":"2021-07-02T12:07:01.255308Z","shell.execute_reply":"2021-07-02T12:07:01.261051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Taking 2000 images for Cross Validation\nimg_test_data = imgdata[:2000]\nimg_test_labels = imglabels[:2000]\n\nimg_test_data.shape, img_test_labels.shape","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:07:02.836456Z","iopub.execute_input":"2021-07-02T12:07:02.836831Z","iopub.status.idle":"2021-07-02T12:07:02.843348Z","shell.execute_reply.started":"2021-07-02T12:07:02.836798Z","shell.execute_reply":"2021-07-02T12:07:02.842352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plotting Dataset\n\nplt.figure(figsize=(6, 6))\nfor i in range(25):\n    plt.subplot(5, 5, i + 1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(img_train_data[i])\n    plt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:09:09.177179Z","iopub.execute_input":"2021-07-02T12:09:09.177512Z","iopub.status.idle":"2021-07-02T12:09:11.258674Z","shell.execute_reply.started":"2021-07-02T12:09:09.177471Z","shell.execute_reply":"2021-07-02T12:09:11.257595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Building an Image Dataset for a Convolutional Neural Network.\ndef convolutional_model(input_shape):\n    model = Sequential()\n    \n    model.add(Conv2D(32, (3,3), activation='relu', input_shape=input_shape,\n                   kernel_initializer=initialize_weights, kernel_regularizer=l2(2e-4)))\n    \n    model.add(MaxPooling2D())\n    \n    model.add(Dropout(0.25))\n    \n    model.add(Conv2D(32, (3,3), activation='relu', kernel_initializer=initialize_weights,\n                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n    \n    model.add(MaxPooling2D())\n    \n    model.add(Dropout(0.25))\n    \n    model.add(Conv2D(32, (3,3), activation='relu', kernel_initializer=initialize_weights,\n                     bias_initializer=initialize_bias, kernel_regularizer=l2(2e-4)))\n    \n    model.add(Flatten())\n    \n    model.add(Dense(1024, activation='relu',\n                   kernel_regularizer=l2(1e-3),\n                   kernel_initializer=initialize_weights,bias_initializer=initialize_bias))\n    \n    model.add(Dense(10, activation='softmax',\n                   kernel_regularizer=l2(1e-3),\n                   kernel_initializer=initialize_weights,bias_initializer=initialize_bias))\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:09:13.258100Z","iopub.execute_input":"2021-07-02T12:09:13.258407Z","iopub.status.idle":"2021-07-02T12:09:13.266650Z","shell.execute_reply.started":"2021-07-02T12:09:13.258381Z","shell.execute_reply":"2021-07-02T12:09:13.265464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = convolutional_model((28,28,1))\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:09:19.068624Z","iopub.execute_input":"2021-07-02T12:09:19.068988Z","iopub.status.idle":"2021-07-02T12:09:19.158669Z","shell.execute_reply.started":"2021-07-02T12:09:19.068958Z","shell.execute_reply":"2021-07-02T12:09:19.157759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Hyperparameters\nlr = 0.0003\nepochs = 10\nbatch_size = 128\n\noptimizer = Adam(lr)\nmodel.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\nhistory = model.fit(img_train_data, img_train_labels, epochs=epochs, batch_size=batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:09:22.803302Z","iopub.execute_input":"2021-07-02T12:09:22.803643Z","iopub.status.idle":"2021-07-02T12:11:51.907001Z","shell.execute_reply.started":"2021-07-02T12:09:22.803613Z","shell.execute_reply":"2021-07-02T12:11:51.906237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(img_test_data, img_test_labels)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:12:01.740733Z","iopub.execute_input":"2021-07-02T12:12:01.741419Z","iopub.status.idle":"2021-07-02T12:12:02.281300Z","shell.execute_reply.started":"2021-07-02T12:12:01.741373Z","shell.execute_reply":"2021-07-02T12:12:02.280359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading Test Dataset\ntest = pd.read_csv('../input/digit-recognizer/test.csv')\ntest = np.reshape(np.array(test), (test.shape[0], 28, 28, 1))","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:12:30.924671Z","iopub.execute_input":"2021-07-02T12:12:30.925035Z","iopub.status.idle":"2021-07-02T12:12:32.353935Z","shell.execute_reply.started":"2021-07-02T12:12:30.925002Z","shell.execute_reply":"2021-07-02T12:12:32.353055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making Predictions on existing model.\nprediction = model.predict(test)\n\n# Rounding the prediction and converting float64 output to int64 output.\nprediction = np.array(np.round(prediction), dtype='int64')\n\n# Converting one-hot encoding into label encoding\nprediction = (np.argmax(prediction, axis=1)).reshape(-1, 1)\n\n# Creating a DataFrame similar to Example Submission\nout = [{'ImageId': i+1, 'Label': prediction[i][0]} for i in range(len(prediction))]\n\n# Creating a .csv file from the DataFrame\npd.DataFrame(out).to_csv('submission_v2.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-07-02T12:12:35.802045Z","iopub.execute_input":"2021-07-02T12:12:35.802414Z","iopub.status.idle":"2021-07-02T12:12:40.110243Z","shell.execute_reply.started":"2021-07-02T12:12:35.802378Z","shell.execute_reply":"2021-07-02T12:12:40.109371Z"},"trusted":true},"execution_count":null,"outputs":[]}]}