{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Libraries","metadata":{}},{"cell_type":"code","source":"# from az_common_funcs import *\nimport pandas as pd\nimport numpy as np\nimport random\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom torch.autograd import Variable\n\nimport seaborn as sns\nsns.set_theme(style='darkgrid')","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:51:55.067878Z","start_time":"2022-09-17T17:51:53.669092Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:20.165841Z","iopub.execute_input":"2022-09-18T03:17:20.166229Z","iopub.status.idle":"2022-09-18T03:17:20.174410Z","shell.execute_reply.started":"2022-09-18T03:17:20.166194Z","shell.execute_reply":"2022-09-18T03:17:20.173382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:51:55.778789Z","start_time":"2022-09-17T17:51:55.769143Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:20.176555Z","iopub.execute_input":"2022-09-18T03:17:20.177317Z","iopub.status.idle":"2022-09-18T03:17:20.186991Z","shell.execute_reply.started":"2022-09-18T03:17:20.177278Z","shell.execute_reply":"2022-09-18T03:17:20.185909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Importing Competition DataSets","metadata":{}},{"cell_type":"code","source":"df_main = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf_submission = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:00.824100Z","start_time":"2022-09-17T17:51:58.555056Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:20.188736Z","iopub.execute_input":"2022-09-18T03:17:20.189204Z","iopub.status.idle":"2022-09-18T03:17:23.816630Z","shell.execute_reply.started":"2022-09-18T03:17:20.189169Z","shell.execute_reply":"2022-09-18T03:17:23.815576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing and Data Exploration","metadata":{}},{"cell_type":"code","source":"# index_to_split_on = list(range(df_main.shape[0]))\n# random.shuffle(a)\n\nrandom.seed(30)\nindex_to_split_on = random.sample(range(df_main.shape[0]), k = int(0.9 * df_main.shape[0]))","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:00.958629Z","start_time":"2022-09-17T17:52:00.942669Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:23.818038Z","iopub.execute_input":"2022-09-18T03:17:23.818490Z","iopub.status.idle":"2022-09-18T03:17:23.859970Z","shell.execute_reply.started":"2022-09-18T03:17:23.818453Z","shell.execute_reply":"2022-09-18T03:17:23.859045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_main.iloc[index_to_split_on].reset_index(drop = True)\ndf_test = df_main.drop(index_to_split_on, axis = 0).reset_index(drop = True)","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:01.277309Z","start_time":"2022-09-17T17:52:01.007557Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:23.861297Z","iopub.execute_input":"2022-09-18T03:17:23.861681Z","iopub.status.idle":"2022-09-18T03:17:24.186495Z","shell.execute_reply.started":"2022-09-18T03:17:23.861629Z","shell.execute_reply":"2022-09-18T03:17:24.185119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df_train.iloc[:,0].to_frame()","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:01.372458Z","start_time":"2022-09-17T17:52:01.370298Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:24.188045Z","iopub.execute_input":"2022-09-18T03:17:24.188665Z","iopub.status.idle":"2022-09-18T03:17:24.200253Z","shell.execute_reply.started":"2022-09-18T03:17:24.188604Z","shell.execute_reply":"2022-09-18T03:17:24.199011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['pixels'] = list(df_train.iloc[:,1:].values)","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:01.900122Z","start_time":"2022-09-17T17:52:01.872885Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:24.201994Z","iopub.execute_input":"2022-09-18T03:17:24.202689Z","iopub.status.idle":"2022-09-18T03:17:24.231871Z","shell.execute_reply.started":"2022-09-18T03:17:24.202602Z","shell.execute_reply":"2022-09-18T03:17:24.230142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:02.696701Z","start_time":"2022-09-17T17:52:02.666697Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:24.234260Z","iopub.execute_input":"2022-09-18T03:17:24.239383Z","iopub.status.idle":"2022-09-18T03:17:24.262156Z","shell.execute_reply.started":"2022-09-18T03:17:24.239335Z","shell.execute_reply":"2022-09-18T03:17:24.260868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# rows = iter(train[train.label == 0].index)\nrows = iter(train.index)","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:03.312275Z","start_time":"2022-09-17T17:52:03.307164Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:24.263776Z","iopub.execute_input":"2022-09-18T03:17:24.264371Z","iopub.status.idle":"2022-09-18T03:17:24.268991Z","shell.execute_reply.started":"2022-09-18T03:17:24.264333Z","shell.execute_reply":"2022-09-18T03:17:24.267930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# row = 45\n# plt.title(f\"Label == {train.loc[row,'label']}\")\n# plt.imshow(train.loc[row,'pixels'].reshape(28,28,1))\n\nplt.figure(figsize=(10,10)) # specifying the overall grid size\n\nfor i in range(25):\n    plt.subplot(5,5,i+1)    # the number of images in the grid is 5*5 (25)\n    row = next(rows)\n    plt.title(f\"Label == {train.loc[row,'label']}\", y = -0.2)\n    plt.imshow(train.loc[row,'pixels'].reshape(28,28,1))\n    plt.axis('off')\n\n    \nplt.subplots_adjust(left=0.1,\n                    bottom=0.1, \n                    right=0.9, \n                    top=0.9, \n                    wspace=0.2, \n                    hspace=0.2)\nplt.show()","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:04.397594Z","start_time":"2022-09-17T17:52:04.070770Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:24.278121Z","iopub.execute_input":"2022-09-18T03:17:24.278853Z","iopub.status.idle":"2022-09-18T03:17:25.336115Z","shell.execute_reply.started":"2022-09-18T03:17:24.278810Z","shell.execute_reply":"2022-09-18T03:17:25.335174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model 2 Training and Results\n1. Model : CNN\n2. Padding : True\n3. Pooling : True\n4. Layers : 8 Layers\n5. Dropout : 0.1","metadata":{}},{"cell_type":"code","source":"class MyDataset():\n    def __init__(self, df_input):\n        \n        x = df_input.iloc[:,1:].values\n        y = df_input.iloc[:,0].values\n        \n        self.x_train = torch.tensor(x, dtype=torch.float32)\n        self.y_train = torch.tensor(y, dtype=torch.int32)\n        \n    def __len__(self):\n        return len(self.y_train)\n    \n    def __getitem__(self,idx):\n        return self.x_train[idx], self.y_train[idx]\n    \n    def __shape_of_xy__(self):\n        print(f\"size of x : {self.x_train.size()}\")\n        print(f\"size of y : {self.y_train.size()}\")","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:06.285135Z","start_time":"2022-09-17T17:52:06.274493Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.340234Z","iopub.execute_input":"2022-09-18T03:17:25.348570Z","iopub.status.idle":"2022-09-18T03:17:25.358289Z","shell.execute_reply.started":"2022-09-18T03:17:25.348517Z","shell.execute_reply":"2022-09-18T03:17:25.357303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_accuracy(Y, Y_pred):\n    Y_pred = Y_pred.data.max(dim=1)[1]\n    \n    acc = (Y == Y_pred).float().mean() \n    \n    # Explanation Example\n    # (torch.tensor(np.array([1,2,3,4,5])) == torch.tensor(np.array([1,2,4,6,5]))).float().mean().item()\n    \n    return acc.item()","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:06.793369Z","start_time":"2022-09-17T17:52:06.788068Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.372339Z","iopub.execute_input":"2022-09-18T03:17:25.374538Z","iopub.status.idle":"2022-09-18T03:17:25.382473Z","shell.execute_reply.started":"2022-09-18T03:17:25.374497Z","shell.execute_reply":"2022-09-18T03:17:25.381347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_accuracy_no_calc(Y, Y_pred):\n    \n    acc = (Y == Y_pred).float().mean() \n    \n    # Explanation Example\n    # (torch.tensor(np.array([1,2,3,4,5])) == torch.tensor(np.array([1,2,4,6,5]))).float().mean().item()\n    \n    return acc.item()","metadata":{"execution":{"iopub.status.busy":"2022-09-18T03:17:25.387170Z","iopub.execute_input":"2022-09-18T03:17:25.389974Z","iopub.status.idle":"2022-09-18T03:17:25.397287Z","shell.execute_reply.started":"2022-09-18T03:17:25.389938Z","shell.execute_reply":"2022-09-18T03:17:25.396334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_2(torch.nn.Module):\n    \n    def __init__(self, keep_prob):\n        \n        super(CNN_2, self).__init__()\n        \n        self.keep_prob = keep_prob\n        # Layer 1\n        # I : (?, 1, 28, 28)\n        # O : (?, 32, 14,14)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_1 = torch.nn.Sequential(torch.nn.Conv2d(1, 32, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 2\n        # I : (?, 32, 14, 14)\n        # O : (?, 64, 7, 7)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_2 = torch.nn.Sequential(torch.nn.Conv2d(32, 64, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 3\n        # I : (?, 64, 7, 7)\n        # O : (?, 128, 7, 7)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_3 = torch.nn.Sequential(torch.nn.Conv2d(64, 128, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 4\n        # I : (?, 128, 7, 7)\n        # O : (?, 256, 2, 2)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_4 = torch.nn.Sequential(torch.nn.Conv2d(128, 256, kernel_size = 3, padding = 1, stride = 2),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 5\n        # I : (?, 256, 2, 2)\n        # O : (?, 1024)\n        self.flatten_layer_1 = torch.nn.Sequential(torch.nn.Flatten())\n        \n        \n        # Layer 6\n        # I : (?, 1024)\n        # O : (?, 512)\n        self.fc_layer_1 = torch.nn.Sequential(torch.nn.Linear(1024, 512),\n                                              torch.nn.ReLU(),\n                                              torch.nn.Dropout(p = 1 - self.keep_prob)\n                                             )\n\n        # Layer 7\n        # I : (?, 512)\n        # O : (?, 250)\n        self.fc_layer_2 = torch.nn.Sequential(torch.nn.Linear(512, 250),\n                                              torch.nn.ReLU(),\n                                              torch.nn.Dropout(p = 1 - self.keep_prob)\n                                             )\n        \n        # Layer 8\n        # I : (?, 512)\n        # O : (?, 250)\n        self.fc_layer_3 = torch.nn.Sequential(torch.nn.Linear(250, 10))\n        \n    \n    def forward(self,x):\n        a = self.cnn_layer_1(x)\n        a = self.cnn_layer_2(a)\n        a = self.cnn_layer_3(a)\n        a = self.cnn_layer_4(a)\n        a = self.flatten_layer_1(a)\n        a = self.fc_layer_1(a)\n        a = self.fc_layer_2(a)\n        a = self.fc_layer_3(a)\n        \n        return a","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:07.591693Z","start_time":"2022-09-17T17:52:07.568038Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.402301Z","iopub.execute_input":"2022-09-18T03:17:25.405042Z","iopub.status.idle":"2022-09-18T03:17:25.425982Z","shell.execute_reply.started":"2022-09-18T03:17:25.404976Z","shell.execute_reply":"2022-09-18T03:17:25.425070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CNN_2(0.9)\nmodel.to(device)\nmodel","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:09.280389Z","start_time":"2022-09-17T17:52:09.251339Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.431275Z","iopub.execute_input":"2022-09-18T03:17:25.434399Z","iopub.status.idle":"2022-09-18T03:17:25.465688Z","shell.execute_reply.started":"2022-09-18T03:17:25.434360Z","shell.execute_reply":"2022-09-18T03:17:25.464877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_set = MyDataset(df_train)","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:20.402131Z","start_time":"2022-09-17T17:52:20.350431Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.470486Z","iopub.execute_input":"2022-09-18T03:17:25.472590Z","iopub.status.idle":"2022-09-18T03:17:25.592303Z","shell.execute_reply.started":"2022-09-18T03:17:25.472553Z","shell.execute_reply":"2022-09-18T03:17:25.591080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\ndata_loader = torch.utils.data.DataLoader(dataset = data_set,\n                                          batch_size = batch_size,\n                                          shuffle = True)","metadata":{"ExecuteTime":{"end_time":"2022-09-17T17:52:23.387669Z","start_time":"2022-09-17T17:52:23.382785Z"},"execution":{"iopub.status.busy":"2022-09-18T03:17:25.597477Z","iopub.execute_input":"2022-09-18T03:17:25.599838Z","iopub.status.idle":"2022-09-18T03:17:25.616106Z","shell.execute_reply.started":"2022-09-18T03:17:25.599793Z","shell.execute_reply":"2022-09-18T03:17:25.615085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\"\"********************* Model 2 Training ***************************\"\"\")\nlearning_rate = 0.001\n\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(params = model.parameters(), lr = learning_rate)\n\nepochs = 15 \ntrain_acc = []\ntrain_cost = []\n\nfor epoch in range(epochs):\n    avg_cost = 0\n    print(f\"Epoch {epoch + 1} Starting\")\n    for batch_no, (batch_x, batch_y) in enumerate(data_loader, start = 1):\n        if batch_x.size()[0] < batch_size : continue\n        \n        X_train = Variable(batch_x).reshape(batch_size, 1, 28, 28).to(device) # reshaping the input tensor \n        Y_train = Variable(batch_y)\n        Y_train = Y_train.type(torch.LongTensor).to(device)\n        \n        optimizer.zero_grad() # zeroing the gradients\n        \n        hypothesis = model(X_train) # making a forward pass\n        \n        cost = criterion(hypothesis, Y_train) # caculating the cost\n        \n        cost.backward() # making a backward pass\n        \n        optimizer.step() # updating the parameters\n        \n        batch_accuracy = compute_accuracy(Y_train, hypothesis) # Calculating Batch Accuracy\n        \n        train_acc.append(batch_accuracy)\n        train_cost.append(cost.item())\n        \n        if batch_no % 100 == 0 :\n            print(f\"Epoch {epoch + 1} :: Batch {batch_no} done :: Accuracy = {batch_accuracy : .2%}\")\n        \n        avg_cost += cost.item() / batch_size\n        \n    print(f\"Epoch {epoch + 1} Done with Avg Cost = {avg_cost}\")\n","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:11:05.481944Z","start_time":"2022-09-11T03:38:35.358316Z"},"scrolled":true,"execution":{"iopub.status.busy":"2022-09-18T03:17:25.621519Z","iopub.execute_input":"2022-09-18T03:17:25.624187Z","iopub.status.idle":"2022-09-18T03:18:11.959921Z","shell.execute_reply.started":"2022-09-18T03:17:25.624146Z","shell.execute_reply":"2022-09-18T03:18:11.958841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction_function(self, batch_size, model):\n    max_rows = self.shape[0]\n    Y_pred = []\n    \n    iter_length = batch_size\n    \n    i_start = 0\n    i_end = i_start + iter_length\n\n    while i_start != i_end:\n        res = model(torch.tensor(self.iloc[i_start:i_end].values, dtype = torch.float32).reshape(i_end-i_start, 1, 28, 28).to(device)) \n        res = res.data.max(dim=1)[1]\n\n        # res_append\n        Y_pred.extend(res)\n\n        if i_end%(iter_length*50) == 0: print(f\"{i_end} rows done!\")\n\n        # increment the indexes\n        i_start = i_end\n        i_end = i_start + iter_length\n        if i_end > max_rows:\n            i_end = max_rows\n    print(\"All rows done!\")\n    \n    return torch.tensor(Y_pred).to(device)\n\npd.DataFrame.prediction_function = prediction_function","metadata":{"execution":{"iopub.status.busy":"2022-09-18T03:18:11.968083Z","iopub.execute_input":"2022-09-18T03:18:11.968513Z","iopub.status.idle":"2022-09-18T03:18:11.978457Z","shell.execute_reply.started":"2022-09-18T03:18:11.968476Z","shell.execute_reply":"2022-09-18T03:18:11.977458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_train = df_train.iloc[:,1:].prediction_function(64, model)","metadata":{"execution":{"iopub.status.busy":"2022-09-18T03:18:11.980037Z","iopub.execute_input":"2022-09-18T03:18:11.980584Z","iopub.status.idle":"2022-09-18T03:18:13.230888Z","shell.execute_reply.started":"2022-09-18T03:18:11.980533Z","shell.execute_reply":"2022-09-18T03:18:13.229908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Y_pred_train = model_2(torch.tensor(df_train.iloc[:,1:].values, dtype = torch.float32).reshape(df_train.shape[0], 1, 28, 28)) ","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:13:27.664995Z","start_time":"2022-09-11T04:12:20.293109Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.232298Z","iopub.execute_input":"2022-09-18T03:18:13.232699Z","iopub.status.idle":"2022-09-18T03:18:13.238004Z","shell.execute_reply.started":"2022-09-18T03:18:13.232663Z","shell.execute_reply":"2022-09-18T03:18:13.237069Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Y_pred_test = model_2(torch.tensor(df_test.iloc[:,1:].values, dtype = torch.float32).reshape(df_test.shape[0], 1, 28, 28).to(device)).data.max(dim=1)[1]","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:13:35.129777Z","start_time":"2022-09-11T04:13:31.901155Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.239372Z","iopub.execute_input":"2022-09-18T03:18:13.240219Z","iopub.status.idle":"2022-09-18T03:18:13.248717Z","shell.execute_reply.started":"2022-09-18T03:18:13.240183Z","shell.execute_reply":"2022-09-18T03:18:13.247810Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Y_pred = Y_pred.data.max(dim=1)[1]","metadata":{"ExecuteTime":{"end_time":"2022-09-11T01:36:32.389394Z","start_time":"2022-09-11T01:36:32.389387Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.250306Z","iopub.execute_input":"2022-09-18T03:18:13.250703Z","iopub.status.idle":"2022-09-18T03:18:13.258886Z","shell.execute_reply.started":"2022-09-18T03:18:13.250633Z","shell.execute_reply":"2022-09-18T03:18:13.257951Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_test = df_test.iloc[:,1:].prediction_function(64, model)","metadata":{"execution":{"iopub.status.busy":"2022-09-18T03:18:13.259991Z","iopub.execute_input":"2022-09-18T03:18:13.260565Z","iopub.status.idle":"2022-09-18T03:18:13.408729Z","shell.execute_reply.started":"2022-09-18T03:18:13.260529Z","shell.execute_reply":"2022-09-18T03:18:13.407629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_train = torch.tensor(df_train.iloc[:,0].values, dtype = torch.int32).type(torch.LongTensor).to(device)","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:13:40.464972Z","start_time":"2022-09-11T04:13:40.453028Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.409887Z","iopub.execute_input":"2022-09-18T03:18:13.410239Z","iopub.status.idle":"2022-09-18T03:18:13.416860Z","shell.execute_reply.started":"2022-09-18T03:18:13.410195Z","shell.execute_reply":"2022-09-18T03:18:13.415717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_test = torch.tensor(df_test.iloc[:,0].values, dtype = torch.int32).type(torch.LongTensor).to(device)","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:13:40.981120Z","start_time":"2022-09-11T04:13:40.963207Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.418459Z","iopub.execute_input":"2022-09-18T03:18:13.419137Z","iopub.status.idle":"2022-09-18T03:18:13.427021Z","shell.execute_reply.started":"2022-09-18T03:18:13.419098Z","shell.execute_reply":"2022-09-18T03:18:13.426047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Accuracy for the train set is = {compute_accuracy_no_calc(Y_train, Y_pred_train) : .2%}\")\nprint(f\"Accuracy for the test set is = {compute_accuracy_no_calc(Y_test, Y_pred_test) : .2%}\")\n# Model1 - 99.46 % and 98.36 %, 1.1% difference\n# Model2 - 99.28 % and 98.81 %, 0.47% difference","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:13:41.978327Z","start_time":"2022-09-11T04:13:41.968282Z"},"execution":{"iopub.status.busy":"2022-09-18T03:18:13.430087Z","iopub.execute_input":"2022-09-18T03:18:13.430741Z","iopub.status.idle":"2022-09-18T03:18:13.437301Z","shell.execute_reply.started":"2022-09-18T03:18:13.430705Z","shell.execute_reply":"2022-09-18T03:18:13.436013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# file_path = f\"models/\"\n# model_name = \"CNN_2\"\n\n# # save\n# torch.save(model_2.state_dict(), f'{file_path}/{model_name}.pt')\n\n# # load\n# # model_2 = CNN_2(0.9)\n# # model_2.load_state_dict(torch.load(f'{file_path}/{model_name}.pt'))","metadata":{"ExecuteTime":{"end_time":"2022-09-11T23:58:10.209441Z","start_time":"2022-09-11T23:58:10.180633Z"},"jupyter":{"source_hidden":true}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(8, 6), dpi=80)\nfig, axs = plt.subplots(ncols=2, figsize=(16, 6), dpi=80)\np1, = axs[0].plot(train_cost, color = 'r')\naxs[0].set_title(\"cost\")\np2, = axs[1].plot(train_acc, color = 'g')\naxs[1].set_title(\"accuracy\")\n# plt.legend([p1,p2], ['cost', 'accuracy'], loc = 'upper right')\nplt.show()","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:14:47.141141Z","start_time":"2022-09-11T04:14:46.942747Z"},"execution":{"iopub.status.busy":"2022-09-18T02:59:17.535600Z","iopub.execute_input":"2022-09-18T02:59:17.535955Z","iopub.status.idle":"2022-09-18T02:59:17.933212Z","shell.execute_reply.started":"2022-09-18T02:59:17.535925Z","shell.execute_reply":"2022-09-18T02:59:17.932196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"row = 1137\n# plt.title(f\"Label == {test.loc[row,'label']}\")\nX_test_sample = torch.tensor(df_test.iloc[row,1:].values, dtype = torch.float32).reshape(1,1,28,28).to(device)\ntest_label = model(X_test_sample).data.max(dim=1)[1].item()\nplt.title(f\"True_Label == {df_test.iloc[row,0]} \\n Predicted Label == {test_label}\")\nplt.imshow(df_test.iloc[row,1:].values.reshape(28,28,1))","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:15:21.349356Z","start_time":"2022-09-11T04:15:21.185649Z"},"execution":{"iopub.status.busy":"2022-09-18T03:00:19.637448Z","iopub.execute_input":"2022-09-18T03:00:19.637814Z","iopub.status.idle":"2022-09-18T03:00:19.909081Z","shell.execute_reply.started":"2022-09-18T03:00:19.637785Z","shell.execute_reply":"2022-09-18T03:00:19.908139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = df_test.iloc[:,0].to_frame()\ntest['pixels'] = list(df_test.iloc[:,1:].values)","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:16:12.479736Z","start_time":"2022-09-11T04:16:12.468688Z"},"execution":{"iopub.status.busy":"2022-09-18T03:00:41.213850Z","iopub.execute_input":"2022-09-18T03:00:41.214257Z","iopub.status.idle":"2022-09-18T03:00:41.223445Z","shell.execute_reply.started":"2022-09-18T03:00:41.214206Z","shell.execute_reply":"2022-09-18T03:00:41.222484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['pred_label_model_2'] = df_test.iloc[:,1:].prediction_function(64, model).tolist()","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:16:27.485945Z","start_time":"2022-09-11T04:16:24.494412Z"},"execution":{"iopub.status.busy":"2022-09-18T03:02:32.226134Z","iopub.execute_input":"2022-09-18T03:02:32.226837Z","iopub.status.idle":"2022-09-18T03:02:32.376431Z","shell.execute_reply.started":"2022-09-18T03:02:32.226802Z","shell.execute_reply":"2022-09-18T03:02:32.375308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rows = iter(test.query(\"(label != pred_label_model_2) and label == 2\").index)\n# rows = iter(test.index)\n# print(list(rows), len(list(rows)))","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:30:03.860624Z","start_time":"2022-09-11T04:30:03.850286Z"},"execution":{"iopub.status.busy":"2022-09-18T03:03:39.197863Z","iopub.execute_input":"2022-09-18T03:03:39.198238Z","iopub.status.idle":"2022-09-18T03:03:39.206824Z","shell.execute_reply.started":"2022-09-18T03:03:39.198190Z","shell.execute_reply":"2022-09-18T03:03:39.205722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,10)) # specifying the overall grid size\n\nfor i in range(25):\n    row = next(rows,\"invalid\")\n    if row != 'invalid' :\n#     X_test_sample = torch.tensor(test.loc[row, 'pixels'], dtype = torch.float32).reshape(1,1,28,28)\n#     predicted_label_sample = model(X_test_sample).max(dim = 1)[1]\n\n        plt.subplot(5,5,i+1) # the number of images in the grid is 5*5 (25)\n        plt.title(f\"Label == {test.loc[row,'label']} \\n\\\n        Predicted_Label_M2 == {test.loc[row,'pred_label_model_2']}\", y = -0.3)\n\n        plt.imshow(test.loc[row,'pixels'].reshape(28,28,1))\n    plt.axis('off')\n\n    \nplt.subplots_adjust(left=0.2,\n                    bottom=0.2, \n                    right=1.4, \n                    top=1.4, \n                    wspace=0.1, \n                    hspace=0.1)\nplt.show()","metadata":{"ExecuteTime":{"end_time":"2022-09-11T04:30:05.052575Z","start_time":"2022-09-11T04:30:04.914812Z"},"code_folding":[],"execution":{"iopub.status.busy":"2022-09-18T03:03:40.752751Z","iopub.execute_input":"2022-09-18T03:03:40.753118Z","iopub.status.idle":"2022-09-18T03:03:40.942560Z","shell.execute_reply.started":"2022-09-18T03:03:40.753084Z","shell.execute_reply":"2022-09-18T03:03:40.941298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusions and Submission","metadata":{"ExecuteTime":{"end_time":"2022-09-11T22:32:58.846557Z","start_time":"2022-09-11T22:32:58.785824Z"}}},{"cell_type":"code","source":"print(f\"Total Parameters in Model          : {sum(p.numel() for p in model.parameters()) : ,}\")\nprint(f\"Total Trainable Parameters in Model 2 : {sum(p.numel() for p in model.parameters() if p.requires_grad) : ,}\")","metadata":{"ExecuteTime":{"end_time":"2022-09-11T22:52:30.792668Z","start_time":"2022-09-11T22:52:30.782126Z"},"execution":{"iopub.status.busy":"2022-09-18T03:04:01.550010Z","iopub.execute_input":"2022-09-18T03:04:01.550397Z","iopub.status.idle":"2022-09-18T03:04:01.556800Z","shell.execute_reply.started":"2022-09-18T03:04:01.550366Z","shell.execute_reply":"2022-09-18T03:04:01.555414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before this architecture, I had tried one with no pooling layer, smaller batch-size and no padding. That architecture had more than 6 M trainable parameters. The architecture is as follows. You can try for yourselves.\n\n```python\nclass CNN_2(torch.nn.Module):\n    \n    def __init__(self, keep_prob):\n        \n        super(CNN_2, self).__init__()\n        \n        self.keep_prob = keep_prob\n        # Layer 1\n        # I : (?, 1, 28, 28)\n        # O : (?, 32, 14,14)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_1 = torch.nn.Sequential(torch.nn.Conv2d(1, 32, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 2\n        # I : (?, 32, 14, 14)\n        # O : (?, 64, 7, 7)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_2 = torch.nn.Sequential(torch.nn.Conv2d(32, 64, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 3\n        # I : (?, 64, 7, 7)\n        # O : (?, 128, 7, 7)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_3 = torch.nn.Sequential(torch.nn.Conv2d(64, 128, kernel_size = 3, padding = 1, stride = 1),\n                                               torch.nn.ReLU(),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 4\n        # I : (?, 128, 7, 7)\n        # O : (?, 256, 2, 2)\n        # size(o) : (n + 2p - f)/s + 1\n        self.cnn_layer_4 = torch.nn.Sequential(torch.nn.Conv2d(128, 256, kernel_size = 3, padding = 1, stride = 2),\n                                               torch.nn.ReLU(),\n                                               torch.nn.MaxPool2d(kernel_size = 2, stride = 2),\n                                               torch.nn.Dropout(p = 1 - self.keep_prob)\n                                              )\n        \n        # Layer 5\n        # I : (?, 256, 2, 2)\n        # O : (?, 1024)\n        self.flatten_layer_1 = torch.nn.Sequential(torch.nn.Flatten())\n        \n        \n        # Layer 6\n        # I : (?, 1024)\n        # O : (?, 512)\n        self.fc_layer_1 = torch.nn.Sequential(torch.nn.Linear(1024, 512),\n                                              torch.nn.ReLU(),\n                                              torch.nn.Dropout(p = 1 - self.keep_prob)\n                                             )\n\n        # Layer 7\n        # I : (?, 512)\n        # O : (?, 250)\n        self.fc_layer_2 = torch.nn.Sequential(torch.nn.Linear(512, 250),\n                                              torch.nn.ReLU(),\n                                              torch.nn.Dropout(p = 1 - self.keep_prob)\n                                             )\n        \n        # Layer 8\n        # I : (?, 512)\n        # O : (?, 250)\n        self.fc_layer_3 = torch.nn.Sequential(torch.nn.Linear(250, 10))\n        \n    \n    def forward(self,x):\n        a = self.cnn_layer_1(x)\n        a = self.cnn_layer_2(a)\n        a = self.cnn_layer_3(a)\n        a = self.cnn_layer_4(a)\n        a = self.flatten_layer_1(a)\n        a = self.fc_layer_1(a)\n        a = self.fc_layer_2(a)\n        a = self.fc_layer_3(a)\n        \n        return a\n```\n\nFollowing conclusions can be derived from the second model:\n1. Adding some elemets (pooling layers) greatly reduced the size of architecture, thereby reducing the number of trainable parameters\n2. Smaller architecture helped in a reduction in training time from 1hr 54mins to 32 mins on CPU ( Here I have used GPU so the difference is little)\n3. Adding dropout in various layers reduced overfitting as the new model has only a differenceof 0.47% between test and train as compared to the older's 1.1%\n4. Overall, the second model is better as it is less overfitting\n\nSome more things that can be tried:\n1. Further reducing the network size\n2. Experimenting with initialization of weights \n3. Addition of Validation Set in the training \n\nReferences: I have followed this [Tutorial](https://towardsdatascience.com/convolutional-neural-network-for-image-classification-with-implementation-on-python-using-pytorch-7b88342c9ca9) by Amine Hadj-Youcef.","metadata":{"ExecuteTime":{"end_time":"2022-09-11T22:46:01.525885Z","start_time":"2022-09-11T22:46:01.513665Z"}}},{"cell_type":"code","source":"submission_pred = df_submission.prediction_function(64, model).tolist()","metadata":{"ExecuteTime":{"end_time":"2022-09-11T22:58:58.441546Z","start_time":"2022-09-11T22:58:08.902083Z"},"execution":{"iopub.status.busy":"2022-09-18T03:10:41.164003Z","iopub.execute_input":"2022-09-18T03:10:41.164397Z","iopub.status.idle":"2022-09-18T03:10:42.337159Z","shell.execute_reply.started":"2022-09-18T03:10:41.164363Z","shell.execute_reply":"2022-09-18T03:10:42.336135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_to_submit = pd.DataFrame(submission_pred, columns = ['Label'])\ndf_to_submit = df_to_submit.rename_axis('ImageId', axis = 0).reset_index()\ndf_to_submit['ImageId'] = df_to_submit['ImageId'] + 1\ndf_to_submit.head()","metadata":{"ExecuteTime":{"end_time":"2022-09-11T23:09:26.119478Z","start_time":"2022-09-11T23:09:26.090329Z"},"execution":{"iopub.status.busy":"2022-09-18T03:10:57.581467Z","iopub.execute_input":"2022-09-18T03:10:57.582047Z","iopub.status.idle":"2022-09-18T03:10:57.606090Z","shell.execute_reply.started":"2022-09-18T03:10:57.582011Z","shell.execute_reply":"2022-09-18T03:10:57.605257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_to_submit.to_csv(\"./submission.csv\", index = False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}