{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nIn this notebook I will examine the effect of regularization techniques on the output.\n\n1. [Dataset](#1)\n    1. [Preparation](#1.1)\n        1. [Normalization](#1.1.1)\n        1. [Reshaping](#1.1.2)\n        1. [Train - validation split](#1.1.3)\n1. [Core architecture](#2)\n1. [Regularization techniques](#3)\n    1. [Dropout](#3.1)\n        1. [Architecture 1](#3.1.1)\n        1. [Architecture 2](#3.1.2)\n        1. [Architecture 3](#3.1.3)\n        1. [Results](#3.1.4)\n    1. [Batch normalization](#3.2)\n        1. [Architecture 1](#3.2.1)\n        1. [Batch normalization + dropout](#3.2.2)\n    1. [Layer normalization](#3.3)\n    1. [Lasso, ridge and elastic regularization](#3.4)\n        1. [L2 regularization](#3.4.1)\n        1. [L1 regularization](#3.4.2)\n        1. [Elastic regularization](#3.4.3)\n    1. [Data augmentation](#3.5)\n1. [Final architecture](#4)","metadata":{}},{"cell_type":"markdown","source":"# Dataset <a id=1></a>\n\nLets quickly check the dataset first.\n\nThis dataset is well-known and well-studied dataset and 101 of the computer vision. For more information, please check the ","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('ggplot')\nimport seaborn as sns\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))        ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-09T07:05:53.807677Z","iopub.execute_input":"2023-09-09T07:05:53.808054Z","iopub.status.idle":"2023-09-09T07:05:55.128961Z","shell.execute_reply.started":"2023-09-09T07:05:53.808022Z","shell.execute_reply":"2023-09-09T07:05:55.128020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndf_test = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:05:55.131167Z","iopub.execute_input":"2023-09-09T07:05:55.131826Z","iopub.status.idle":"2023-09-09T07:06:00.599281Z","shell.execute_reply.started":"2023-09-09T07:05:55.131794Z","shell.execute_reply":"2023-09-09T07:06:00.598254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_train.info()) # 42000 entries, 784 pixels on each image, and 1 label\n\nprint(\"-\"*40)\n\nprint(df_test.info()) # 28000 entries, 784 pixels, no label for obvious reasons","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:00.600523Z","iopub.execute_input":"2023-09-09T07:06:00.600817Z","iopub.status.idle":"2023-09-09T07:06:00.689348Z","shell.execute_reply.started":"2023-09-09T07:06:00.600793Z","shell.execute_reply":"2023-09-09T07:06:00.688115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:00.691574Z","iopub.execute_input":"2023-09-09T07:06:00.691902Z","iopub.status.idle":"2023-09-09T07:06:00.716172Z","shell.execute_reply.started":"2023-09-09T07:06:00.691872Z","shell.execute_reply":"2023-09-09T07:06:00.714920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = df_train.iloc[:, 1:].values\ny_train = df_train[\"label\"].values\nX_test = df_test.values","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:00.717455Z","iopub.execute_input":"2023-09-09T07:06:00.717852Z","iopub.status.idle":"2023-09-09T07:06:00.724236Z","shell.execute_reply.started":"2023-09-09T07:06:00.717823Z","shell.execute_reply":"2023-09-09T07:06:00.722400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(X_train), X_train.shape)\n\nprint(type(y_train), y_train.shape)\n\nprint(type(X_test), X_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:00.726078Z","iopub.execute_input":"2023-09-09T07:06:00.726456Z","iopub.status.idle":"2023-09-09T07:06:00.741054Z","shell.execute_reply.started":"2023-09-09T07:06:00.726426Z","shell.execute_reply":"2023-09-09T07:06:00.739874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"lets check one image","metadata":{}},{"cell_type":"code","source":"plt.imshow(X_train[10].reshape(28, 28), cmap=\"gray\")\nplt.title(y_train[10])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:00.742637Z","iopub.execute_input":"2023-09-09T07:06:00.743034Z","iopub.status.idle":"2023-09-09T07:06:01.012726Z","shell.execute_reply.started":"2023-09-09T07:06:00.742988Z","shell.execute_reply":"2023-09-09T07:06:01.011939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values, counts = np.unique(y_train, return_counts=True)\n\nplt.figure()\ng = sns.countplot(x = y_train, palette=\"Spectral\")\nplt.title(\"Number of classes in train set\")\nplt.show()\n\nprint(\"-\"*40)\n\nprint(counts)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:01.013745Z","iopub.execute_input":"2023-09-09T07:06:01.014443Z","iopub.status.idle":"2023-09-09T07:06:01.240891Z","shell.execute_reply.started":"2023-09-09T07:06:01.014410Z","shell.execute_reply":"2023-09-09T07:06:01.239519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation <a id=1.1></a>\n# Normalization <a id=1.1.1></a>\nFor more details you can check my notebook about normalization [here](https://www.kaggle.com/code/atuzen/dive-into-normalization/) and [here](https://www.kaggle.com/code/atuzen/dive-into-normalization-2/) .","metadata":{}},{"cell_type":"code","source":"X_train = X_train / 255\nX_test = X_test / 255","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:01.242102Z","iopub.execute_input":"2023-09-09T07:06:01.242456Z","iopub.status.idle":"2023-09-09T07:06:01.476966Z","shell.execute_reply.started":"2023-09-09T07:06:01.242426Z","shell.execute_reply":"2023-09-09T07:06:01.475879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reshaping <a id=1.1.2></a>","metadata":{}},{"cell_type":"code","source":"X_train = X_train.reshape(-1, 28, 28)\nX_test  = X_test.reshape(-1, 28, 28)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:01.479617Z","iopub.execute_input":"2023-09-09T07:06:01.479953Z","iopub.status.idle":"2023-09-09T07:06:01.485150Z","shell.execute_reply.started":"2023-09-09T07:06:01.479923Z","shell.execute_reply":"2023-09-09T07:06:01.484001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train - validation split <a id=1.1.3></a>","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size = 0.2, random_state=0)\nX_train.shape, X_val.shape, y_train.shape, y_val.shape","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:01.486496Z","iopub.execute_input":"2023-09-09T07:06:01.486775Z","iopub.status.idle":"2023-09-09T07:06:02.112162Z","shell.execute_reply.started":"2023-09-09T07:06:01.486751Z","shell.execute_reply":"2023-09-09T07:06:02.110913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Core architecture <a id=2></a>\n\nNow, dataset is ready for us to work on.\n\n***Please note that our purpose here is examining the regularization techniques, we are not here to hit perfect accuracies. Therefore our base architecture will be quite simple and number of epochs will be rather low.***","metadata":{}},{"cell_type":"code","source":"# Pytorch libraries\n\nimport torch\nimport torch.nn as nn","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:02.116014Z","iopub.execute_input":"2023-09-09T07:06:02.116347Z","iopub.status.idle":"2023-09-09T07:06:05.129732Z","shell.execute_reply.started":"2023-09-09T07:06:02.116318Z","shell.execute_reply":"2023-09-09T07:06:05.128571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## This is our base architecture. we will make changes according to this architecture.\n\nclass CNN_no_reg(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:05.131117Z","iopub.execute_input":"2023-09-09T07:06:05.131620Z","iopub.status.idle":"2023-09-09T07:06:05.139726Z","shell.execute_reply.started":"2023-09-09T07:06:05.131590Z","shell.execute_reply":"2023-09-09T07:06:05.138904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = torch.FloatTensor(X_train)\nX_val = torch.FloatTensor(X_val)\ny_train = torch.tensor(y_train, dtype=torch.long)\ny_val = torch.tensor(y_val, dtype=torch.long)\nX_test = torch.FloatTensor(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:05.141111Z","iopub.execute_input":"2023-09-09T07:06:05.141504Z","iopub.status.idle":"2023-09-09T07:06:05.296252Z","shell.execute_reply.started":"2023-09-09T07:06:05.141468Z","shell.execute_reply":"2023-09-09T07:06:05.295367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"base_model = CNN_no_reg()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(base_model.parameters(), lr=1e-3)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:52:16.752186Z","iopub.execute_input":"2023-09-07T04:52:16.752546Z","iopub.status.idle":"2023-09-07T04:52:16.773291Z","shell.execute_reply.started":"2023-09-07T04:52:16.752515Z","shell.execute_reply":"2023-09-07T04:52:16.771793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Lets train our core model and test the accuracy in validation set.**","metadata":{}},{"cell_type":"code","source":"num_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    base_model.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = base_model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    base_model.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = base_model(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:52:16.777242Z","iopub.execute_input":"2023-09-07T04:52:16.777634Z","iopub.status.idle":"2023-09-07T04:55:03.450532Z","shell.execute_reply.started":"2023-09-07T04:52:16.777603Z","shell.execute_reply":"2023-09-07T04:55:03.449038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Lets create submission.csv file for base model.**\n\n*Test accuracy for this core architecture -> 0.98542*","metadata":{}},{"cell_type":"code","source":"predictions = []\n\nbase_model.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = base_model(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_base.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:55:03.451842Z","iopub.execute_input":"2023-09-07T04:55:03.452168Z","iopub.status.idle":"2023-09-07T04:55:05.247691Z","shell.execute_reply.started":"2023-09-07T04:55:03.452139Z","shell.execute_reply":"2023-09-07T04:55:05.246740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Regularization techniques <a id=3></a>\n\nNow, it is time to make some regularization to enchance our model's accuracy.","metadata":{}},{"cell_type":"markdown","source":"# Dropout <a id=3.1></a>\n\nDuring training dropout randomly makes some neurons to inactive. This will prevent dependencies on individual neurons and we will have a more robust model. We can apply this operation to fc and conv layers.","metadata":{}},{"cell_type":"markdown","source":"### Architecture 1 <a id=3.1.1></a>","metadata":{}},{"cell_type":"code","source":"class CNN_dropout_conv(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.dropout1 = nn.Dropout2d(0.25)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.dropout2 = nn.Dropout2d(0.25)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.dropout1(h)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.dropout2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:55:05.249469Z","iopub.execute_input":"2023-09-07T04:55:05.250141Z","iopub.status.idle":"2023-09-07T04:55:05.262529Z","shell.execute_reply.started":"2023-09-07T04:55:05.250097Z","shell.execute_reply":"2023-09-07T04:55:05.261406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_drop_1 = CNN_dropout_conv()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_drop_1.parameters(), lr=1e-3)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:55:05.264084Z","iopub.execute_input":"2023-09-07T04:55:05.264407Z","iopub.status.idle":"2023-09-07T04:55:05.284258Z","shell.execute_reply.started":"2023-09-07T04:55:05.264380Z","shell.execute_reply":"2023-09-07T04:55:05.283106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_drop_1.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_drop_1(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_drop_1.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_drop_1(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\n    predictions = []\n\nmodel_drop_1.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_drop_1(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_drop_1.csv\", index=False)\n\n# Final score 0.9885 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:55:05.288332Z","iopub.execute_input":"2023-09-07T04:55:05.288682Z","iopub.status.idle":"2023-09-07T04:57:54.334635Z","shell.execute_reply.started":"2023-09-07T04:55:05.288656Z","shell.execute_reply":"2023-09-07T04:57:54.333656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Architecture 2 <a id=3.1.2></a>","metadata":{}},{"cell_type":"code","source":"class CNN_dropout_fc(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        self.dropout = nn.Dropout(0.5)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.dropout(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:57:54.335704Z","iopub.execute_input":"2023-09-07T04:57:54.335971Z","iopub.status.idle":"2023-09-07T04:57:54.343527Z","shell.execute_reply.started":"2023-09-07T04:57:54.335948Z","shell.execute_reply":"2023-09-07T04:57:54.342669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_drop_2 = CNN_dropout_fc()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_drop_2.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_drop_2.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_drop_2(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_drop_2.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_drop_2(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\n    predictions = []\n\nmodel_drop_2.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_drop_2(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_drop_2.csv\", index=False)\n\n# Final score 0.98925 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T04:57:54.344642Z","iopub.execute_input":"2023-09-07T04:57:54.344895Z","iopub.status.idle":"2023-09-07T05:00:46.043782Z","shell.execute_reply.started":"2023-09-07T04:57:54.344873Z","shell.execute_reply":"2023-09-07T05:00:46.042745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Architecture 3 <a id=3.1.3></a>","metadata":{}},{"cell_type":"code","source":"class CNN_dropout(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.dropout1 = nn.Dropout2d(0.25)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.dropout2 = nn.Dropout2d(0.25)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        self.dropout = nn.Dropout(0.5)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.dropout1(h)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.dropout2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.dropout(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:00:46.045173Z","iopub.execute_input":"2023-09-07T05:00:46.045556Z","iopub.status.idle":"2023-09-07T05:00:46.054467Z","shell.execute_reply.started":"2023-09-07T05:00:46.045518Z","shell.execute_reply":"2023-09-07T05:00:46.053374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_dropout = CNN_dropout()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_dropout.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_dropout.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_dropout(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_dropout.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_dropout(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_dropout.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_dropout(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_drop_3.csv\", index=False)\n\n# Final score 0.98846 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:00:46.056246Z","iopub.execute_input":"2023-09-07T05:00:46.056909Z","iopub.status.idle":"2023-09-07T05:03:36.215946Z","shell.execute_reply.started":"2023-09-07T05:00:46.056881Z","shell.execute_reply":"2023-09-07T05:03:36.214864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Results <a id=3.1.4></a>\n\nIn the first architecture, we add two dropout layers after the convolutional layers.\n\nIn the second architecture dropout layer added after fully connected layer.\n\nThe third architecture combines the first and second.","metadata":{}},{"cell_type":"code","source":"# I add the results manually here, I am pretty sure there are better ways to do this. But we have a lot to do still.\n\ny_lims = [0.97, .993]\nresults = [0.98542, 0.9885, 0.98925, 0.98846]\nlabels = [\"Base\", \"Conv Dropout\", \"FC Dropout\", \"Conv + FC dropout\"]\n\nplt.figure()\nplt.title(\"Impact of dropout compared to base architecture\")\nplt.ylabel(\"Accuracy\")\nplt.xticks(fontsize = 8, rotation=45)\nplt.bar(labels, results, label=labels)\nax = plt.gca()\nax.set_ylim(y_lims)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:03:36.217419Z","iopub.execute_input":"2023-09-07T05:03:36.218851Z","iopub.status.idle":"2023-09-07T05:03:36.410537Z","shell.execute_reply.started":"2023-09-07T05:03:36.218813Z","shell.execute_reply":"2023-09-07T05:03:36.409532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**In my experience, dropout shows differences when the number of epochs increases. If you use a low number of epochs, dropout may perform worse.**\n\n**Note that dropout operation is based on randomness, therefore your results may differ from mine.**\n\n**Overall, adding dropout regularization improves the performance on unseen data by a decent amount.**","metadata":{}},{"cell_type":"markdown","source":"# Batch normalization <a id=3.2></a>","metadata":{}},{"cell_type":"markdown","source":"### Architecture 1 <a id=3.2.1></a>","metadata":{}},{"cell_type":"code","source":"class CNN_batch_conv(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.batch1 = nn.BatchNorm2d(16)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.batch2 = nn.BatchNorm2d(32)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.batch1(h)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.batch2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:03:36.411913Z","iopub.execute_input":"2023-09-07T05:03:36.412641Z","iopub.status.idle":"2023-09-07T05:03:36.421424Z","shell.execute_reply.started":"2023-09-07T05:03:36.412610Z","shell.execute_reply":"2023-09-07T05:03:36.420047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_batch_conv = CNN_batch_conv()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_batch_conv.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_batch_conv.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_batch_conv(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_batch_conv.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_batch_conv(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_batch_conv.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_batch_conv(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_batch_1.csv\", index=False)\n\n# Final score 0.98192 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:03:36.423006Z","iopub.execute_input":"2023-09-07T05:03:36.423385Z","iopub.status.idle":"2023-09-07T05:06:54.454463Z","shell.execute_reply.started":"2023-09-07T05:03:36.423350Z","shell.execute_reply":"2023-09-07T05:06:54.453033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Batch normalization + dropout <a id=3.2.2></a>","metadata":{}},{"cell_type":"code","source":"class CNN_batch_dropout(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.batch1 = nn.BatchNorm2d(16)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.batch2 = nn.BatchNorm2d(32)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 10)\n        self.dropout = nn.Dropout(0.5)\n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.batch1(h)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.batch2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.dropout(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:06:54.458997Z","iopub.execute_input":"2023-09-07T05:06:54.459357Z","iopub.status.idle":"2023-09-07T05:06:54.469121Z","shell.execute_reply.started":"2023-09-07T05:06:54.459328Z","shell.execute_reply":"2023-09-07T05:06:54.467777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_batch_drop = CNN_batch_dropout()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_batch_drop.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_batch_drop.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_batch_drop(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_batch_drop.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_batch_drop(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_batch_drop.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_batch_drop(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_batch_2.csv\", index=False)\n\n# Final score 0.98192 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:06:54.470588Z","iopub.execute_input":"2023-09-07T05:06:54.470925Z","iopub.status.idle":"2023-09-07T05:10:12.063777Z","shell.execute_reply.started":"2023-09-07T05:06:54.470896Z","shell.execute_reply":"2023-09-07T05:10:12.062400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Layer normalization <a id=3.3></a>\n\nImplementation of layer normalization is a bit different than others. We will add another fc layer first.","metadata":{}},{"cell_type":"code","source":"class CNN_layer_norm(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 160)\n        self.layernorm = nn.LayerNorm(160)\n        self.relu3 = nn.ReLU() \n        self.fc2 = nn.Linear(160, 10)        \n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.layernorm(h)\n        h = self.relu3(h)\n        h = self.fc2(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:10:12.064913Z","iopub.execute_input":"2023-09-07T05:10:12.065253Z","iopub.status.idle":"2023-09-07T05:10:12.075896Z","shell.execute_reply.started":"2023-09-07T05:10:12.065224Z","shell.execute_reply":"2023-09-07T05:10:12.074249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_layernorm = CNN_layer_norm()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_layernorm.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_layernorm.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_layernorm(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_layernorm.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_layernorm(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_layernorm.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_layernorm(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_layer.csv\", index=False)\n\n# Final score 0.99028 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:10:12.077955Z","iopub.execute_input":"2023-09-07T05:10:12.078316Z","iopub.status.idle":"2023-09-07T05:13:05.437800Z","shell.execute_reply.started":"2023-09-07T05:10:12.078247Z","shell.execute_reply":"2023-09-07T05:13:05.436782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lasso, ridge and elastic regularization <a id=3.4></a>\n\nThese are fancy way to say L1, L2 and combination of L1 and L2 regularization. We will start with L2 as it is already implemented as a parameter in pytorch.","metadata":{}},{"cell_type":"markdown","source":"### L2 regularization <a id=3.4.1></a>","metadata":{}},{"cell_type":"code","source":"class CNN_l2(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 160)\n        self.relu3 = nn.ReLU() \n        self.fc2 = nn.Linear(160, 10)        \n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.relu3(h)\n        h = self.fc2(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:13:05.439039Z","iopub.execute_input":"2023-09-07T05:13:05.439775Z","iopub.status.idle":"2023-09-07T05:13:05.448868Z","shell.execute_reply.started":"2023-09-07T05:13:05.439746Z","shell.execute_reply":"2023-09-07T05:13:05.447403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_l2 = CNN_l2()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_l2.parameters(), lr=1e-3, weight_decay = 1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_l2.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_l2(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_l2.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_l2(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_l2.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_l2(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_l2.csv\", index=False)\n\n# Final score 0.98482 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:13:05.450157Z","iopub.execute_input":"2023-09-07T05:13:05.450442Z","iopub.status.idle":"2023-09-07T05:16:03.106574Z","shell.execute_reply.started":"2023-09-07T05:13:05.450417Z","shell.execute_reply":"2023-09-07T05:16:03.105078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### L1 regularization <a id=3.4.2></a>\n\nL1 regularization is not implemented in Adam optimizer, therefore we will do it ourselves.","metadata":{}},{"cell_type":"code","source":"class CNN_l1(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 160)\n        self.relu3 = nn.ReLU() \n        self.fc2 = nn.Linear(160, 10)        \n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.relu3(h)\n        h = self.fc2(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:16:03.107902Z","iopub.execute_input":"2023-09-07T05:16:03.108216Z","iopub.status.idle":"2023-09-07T05:16:03.117319Z","shell.execute_reply.started":"2023-09-07T05:16:03.108191Z","shell.execute_reply":"2023-09-07T05:16:03.115705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"l1_lambda = 1e-4\n\nmodel_l1 = CNN_l1()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_l1.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_l1.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_l1(inputs)\n        loss = criterion(outputs, labels)\n        \n        l1_reg = 0\n        for param in model_l1.parameters():\n            l1_reg += torch.abs(param).sum() \n\n        loss += l1_lambda * l1_reg        \n        \n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_l1.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_l1(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_l1.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_l1(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_l1.csv\", index=False)\n\n# Final score 0.97989 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:16:03.118358Z","iopub.execute_input":"2023-09-07T05:16:03.118699Z","iopub.status.idle":"2023-09-07T05:19:05.682347Z","shell.execute_reply.started":"2023-09-07T05:16:03.118671Z","shell.execute_reply":"2023-09-07T05:19:05.681320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Elastic net regularization<a id=3.4.3></a>","metadata":{}},{"cell_type":"code","source":"l1_lambda = 1e-4\n\nmodel_l1 = CNN_l1()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_l1.parameters(), lr=1e-3, weight_decay = 1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_l1.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs = X_train[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs = model_l1(inputs)\n        loss = criterion(outputs, labels)\n        \n        l1_reg = 0\n        for param in model_l1.parameters():\n            l1_reg += torch.abs(param).sum() \n\n        loss += l1_lambda * l1_reg        \n        \n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_l1.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_l1(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_l1.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_l1(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_elastic.csv\", index=False)\n\n# Final score 0.9825 on test set.","metadata":{"execution":{"iopub.status.busy":"2023-09-07T05:19:05.683582Z","iopub.execute_input":"2023-09-07T05:19:05.683886Z","iopub.status.idle":"2023-09-07T05:22:08.425644Z","shell.execute_reply.started":"2023-09-07T05:19:05.683860Z","shell.execute_reply":"2023-09-07T05:22:08.424712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data augmentation <a id=3.5></a>\n\nData augmentation is a well-known and highly effective regularization technique.\n\nIn this notebook, we will keep it as simple as we can, just using rotation, horizontal flip (no vertical flip for this dataset). You are welcome to try other methods as well.","metadata":{}},{"cell_type":"code","source":"from torchvision.transforms import v2","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:15.140165Z","iopub.execute_input":"2023-09-09T07:06:15.140873Z","iopub.status.idle":"2023-09-09T07:06:15.709655Z","shell.execute_reply.started":"2023-09-09T07:06:15.140838Z","shell.execute_reply":"2023-09-09T07:06:15.708294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transforms = v2.Compose([\n    v2.RandomRotation(degrees = 20), # Randomly rotates the image in range of (-20, 20) \n    v2.RandomHorizontalFlip(p = 0.3), # Randomly flips the image horizontally\n    v2.ToDtype(torch.float32),\n])\n\nX_train_aug = transforms(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-09-08T17:30:44.613235Z","iopub.execute_input":"2023-09-08T17:30:44.614691Z","iopub.status.idle":"2023-09-08T17:30:45.087827Z","shell.execute_reply.started":"2023-09-08T17:30:44.614631Z","shell.execute_reply":"2023-09-08T17:30:45.086540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_aug(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.relu1 = nn.ReLU()\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.relu2 = nn.ReLU()\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 160)\n        self.relu3 = nn.ReLU() \n        self.fc2 = nn.Linear(160, 10)        \n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.relu1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.relu3(h)\n        h = self.fc2(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-08T17:31:09.619602Z","iopub.execute_input":"2023-09-08T17:31:09.620002Z","iopub.status.idle":"2023-09-08T17:31:09.629732Z","shell.execute_reply.started":"2023-09-08T17:31:09.619974Z","shell.execute_reply":"2023-09-08T17:31:09.628470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_aug = CNN_aug()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_aug.parameters(), lr=1e-3)\n\nnum_epochs = 25\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_aug.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs_orig = X_train[i:i+batch_size].unsqueeze(1)\n        inputs_aug = X_train_aug[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs_orig = model_aug(inputs_orig)        \n        outputs_aug = model_aug(inputs_aug)\n        loss = criterion(outputs_orig, labels) + criterion(outputs_aug, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_aug.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_aug(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_aug.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_aug(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_aug.csv\", index=False)\n\n# Final score 0.97932 on test set, this technique requires a little bit more training.","metadata":{"execution":{"iopub.status.busy":"2023-09-08T17:39:25.037563Z","iopub.execute_input":"2023-09-08T17:39:25.038562Z","iopub.status.idle":"2023-09-08T17:46:27.272943Z","shell.execute_reply.started":"2023-09-08T17:39:25.038520Z","shell.execute_reply":"2023-09-08T17:46:27.271645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final architecture <a id=4></a>\n\nWe examined a lot of regularization techniques all of them increased our accuracy on test set (some required longer trains), some of them in small amount while others enchanced base architecture quite well.\n\nAre we gonna use all of them in the final architecture? No.\n\nSome techniques are better suited for CNN architecture, such as batch normalization is widely used for CNNs and layer normalization is proposed for RNN type of deep models.\n\nAlso, we will skip L1, L2 and elastic regularizations in the final implementation. I am afraid to say this, they are certainly strong methods, but kind of outdated :(.\n\nWe will also tune the data augmentation a little bit down. And increase number of epochs to 40.","metadata":{}},{"cell_type":"code","source":"transforms = v2.Compose([\n    v2.RandomRotation(degrees = 15), # Randomly rotates the image in range of (-15, 15) \n    v2.RandomHorizontalFlip(p = 0.25), # Randomly flips the image horizontally\n    v2.ToDtype(torch.float32),\n])\n\nX_train_aug = transforms(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:19.661806Z","iopub.execute_input":"2023-09-09T07:06:19.662174Z","iopub.status.idle":"2023-09-09T07:06:20.133125Z","shell.execute_reply.started":"2023-09-09T07:06:19.662146Z","shell.execute_reply":"2023-09-09T07:06:20.131849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN_final(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=7, padding=1)\n        self.batch1 = nn.BatchNorm2d(16)\n        self.relu1 = nn.ReLU()\n        self.dropout1 = nn.Dropout2d(0.1)\n        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, padding=1)\n        self.batch2 = nn.BatchNorm2d(32)\n        self.relu2 = nn.ReLU()\n        self.dropout2 = nn.Dropout2d(0.1)\n        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n        self.fc1 = nn.Linear(32 * 5 * 5, 32 * 5)\n        self.dropout3 = nn.Dropout(0.25)\n        self.relu3 = nn.ReLU() \n        self.fc2 = nn.Linear(32 * 5, 10)        \n        \n    def forward(self, x):\n        h = self.conv1(x)\n        h = self.batch1(h)\n        h = self.relu1(h)\n        h = self.dropout1(h)\n        h = self.pool1(h)\n        h = self.conv2(h)\n        h = self.batch2(h)\n        h = self.relu2(h)\n        h = self.pool2(h)\n        h = self.dropout2(h)\n        h = h.view(h.size(0), -1)\n        h = self.fc1(h)\n        h = self.dropout3(h)\n        h = self.relu3(h)\n        h = self.fc2(h)\n        return h","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:36.326661Z","iopub.execute_input":"2023-09-09T07:06:36.327062Z","iopub.status.idle":"2023-09-09T07:06:36.338060Z","shell.execute_reply.started":"2023-09-09T07:06:36.327030Z","shell.execute_reply":"2023-09-09T07:06:36.336899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_final = CNN_final()\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model_final.parameters(), lr=1e-3)\n\nnum_epochs = 40\nbatch_size = 128\n\nfor epoch in range(num_epochs):\n    model_final.train()\n    train_loss = 0.0\n    for i in range(0, len(X_train), batch_size):\n        inputs_orig = X_train[i:i+batch_size].unsqueeze(1)\n        inputs_aug = X_train_aug[i:i+batch_size].unsqueeze(1)\n        labels = y_train[i:i+batch_size]\n        optimizer.zero_grad()\n        outputs_orig = model_final(inputs_orig)        \n        outputs_aug = model_final(inputs_aug)\n        loss = criterion(outputs_orig, labels) + criterion(outputs_aug, labels)\n        loss.backward()\n        optimizer.step()\n        train_loss += loss.item()\n    \n    model_final.eval()\n    val_correct = 0\n    val_total = 0\n    val_loss = 0.0\n    with torch.no_grad():\n        for i in range(0, len(X_val), batch_size):\n            inputs = X_val[i:i+batch_size].unsqueeze(1)\n            labels = y_val[i:i+batch_size]\n            outputs = model_final(inputs)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n            _, predicted = torch.max(outputs, 1)\n            val_total += labels.size(0)\n            val_correct += (predicted == labels).sum().item()\n    \n    print(f\"Epoch {epoch+1}\\tTrain Loss: {(train_loss/len(X_train)):.4f}\\tValidation Loss: {(val_loss/len(X_val)):.4f}\\tValidation Accuracy: {(val_correct/val_total):.4f}\")\n    \n\npredictions = []\n\nmodel_final.eval()\n\nwith torch.no_grad():\n    for i in range(0, len(X_test), batch_size):\n        batch = X_test[i:i+batch_size].unsqueeze(1)\n        outputs = model_final(batch)\n        _, predicted = torch.max(outputs, 1)\n        predictions.extend(predicted.numpy())\n\nsubmission_df = pd.DataFrame({\"ImageId\": range(1, len(X_test) + 1), \"Label\": predictions})\n\nsubmission_df.to_csv(\"submission_final.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-09-09T07:06:37.425272Z","iopub.execute_input":"2023-09-09T07:06:37.425655Z","iopub.status.idle":"2023-09-09T07:41:38.430279Z","shell.execute_reply.started":"2023-09-09T07:06:37.425623Z","shell.execute_reply":"2023-09-09T07:41:38.429359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final submission score: 0.99114\n\n##### I appreciate any feedback. Please let me know if anything is missing or if you would like to add anything. I hope you enjoy this notebook.\n\n##### Links: [GitHub](https://github.com/ahmetTuzen/) and [LinkedIn](https://www.linkedin.com/in/ahmet-tuzen/)","metadata":{}}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}