{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import libraries","metadata":{"id":"3OHqrodZQROP"}},{"cell_type":"code","source":"import time\n\nimport pandas as pd\nimport platform\nimport io\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import cm \n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\n\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torchvision.datasets import CIFAR10\nimport torchvision.transforms as transforms\nimport torchvision\n\nimport matplotlib.pyplot as plt\nimport numpy as np","metadata":{"id":"MyL9SFPWI9gm","execution":{"iopub.status.busy":"2022-05-17T15:30:45.033573Z","iopub.execute_input":"2022-05-17T15:30:45.033916Z","iopub.status.idle":"2022-05-17T15:30:46.823885Z","shell.execute_reply.started":"2022-05-17T15:30:45.033836Z","shell.execute_reply":"2022-05-17T15:30:46.823105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nIMAGE_WIDTH = 32\nIMAGE_HEIGHT = 32\nCOLOR_CHANNELS = 3\nN_CLASSES = 10","metadata":{"id":"033jm0hmXEdU","execution":{"iopub.status.busy":"2022-05-17T15:30:46.825659Z","iopub.execute_input":"2022-05-17T15:30:46.825908Z","iopub.status.idle":"2022-05-17T15:30:46.831284Z","shell.execute_reply.started":"2022-05-17T15:30:46.825873Z","shell.execute_reply":"2022-05-17T15:30:46.829476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check if cuda is available\ncuda = torch.cuda.is_available()\nprint(cuda)","metadata":{"id":"z8yWxOLEw3PE","outputId":"251b8fa7-f09f-4f35-eb83-270ea5699bd6","execution":{"iopub.status.busy":"2022-05-17T15:30:46.833262Z","iopub.execute_input":"2022-05-17T15:30:46.834040Z","iopub.status.idle":"2022-05-17T15:30:46.891528Z","shell.execute_reply.started":"2022-05-17T15:30:46.833995Z","shell.execute_reply":"2022-05-17T15:30:46.890358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load raw CIFAR-10 Dataset and Labels","metadata":{"id":"emPfVTRtQLR1"}},{"cell_type":"code","source":"transform = transforms.Compose(\n    [transforms.ToTensor(),\n     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])","metadata":{"id":"tGO_AC1ZBFa5","execution":{"iopub.status.busy":"2022-05-17T15:30:46.895840Z","iopub.execute_input":"2022-05-17T15:30:46.896250Z","iopub.status.idle":"2022-05-17T15:30:46.902900Z","shell.execute_reply.started":"2022-05-17T15:30:46.896217Z","shell.execute_reply":"2022-05-17T15:30:46.902054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = CIFAR10(root='.',\n                        train=True,\n                        transform=transform,\n                        download=True)\n\ntest_dataset  = CIFAR10(root='.',\n                        train=False,\n                        transform=transform,)\n","metadata":{"id":"DVMG2SCcJIfd","outputId":"69f32745-55ed-42f1-bc79-613569ed333b","execution":{"iopub.status.busy":"2022-05-17T15:30:46.904574Z","iopub.execute_input":"2022-05-17T15:30:46.905221Z","iopub.status.idle":"2022-05-17T15:31:02.303492Z","shell.execute_reply.started":"2022-05-17T15:30:46.905180Z","shell.execute_reply":"2022-05-17T15:31:02.302378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the meta data and batch labels\ndata_batch_1 = pd.read_pickle(r'./cifar-10-batches-py/data_batch_1')\nmeta_data    = pd.read_pickle(r'./cifar-10-batches-py/batches.meta')","metadata":{"id":"4RwhtrUsLLN2","execution":{"iopub.status.busy":"2022-05-17T15:31:02.304854Z","iopub.execute_input":"2022-05-17T15:31:02.305237Z","iopub.status.idle":"2022-05-17T15:31:02.444769Z","shell.execute_reply.started":"2022-05-17T15:31:02.305187Z","shell.execute_reply":"2022-05-17T15:31:02.443973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(data_batch_1.keys())\nprint(meta_data.keys())","metadata":{"id":"ZoAjdPppQxDv","outputId":"15c1924d-e0b2-4c52-ccab-b6552fd66633","execution":{"iopub.status.busy":"2022-05-17T15:31:02.445991Z","iopub.execute_input":"2022-05-17T15:31:02.446353Z","iopub.status.idle":"2022-05-17T15:31:02.451852Z","shell.execute_reply.started":"2022-05-17T15:31:02.446319Z","shell.execute_reply":"2022-05-17T15:31:02.450641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_hot_labels = set(data_batch_1['labels']) # One-hot encoding labels\none_hot_labels","metadata":{"id":"BImsQBjoQ1uf","outputId":"e2d66ad6-cba9-4f29-f5b5-57c9774bcdc2","execution":{"iopub.status.busy":"2022-05-17T15:31:02.453731Z","iopub.execute_input":"2022-05-17T15:31:02.454196Z","iopub.status.idle":"2022-05-17T15:31:02.464461Z","shell.execute_reply.started":"2022-05-17T15:31:02.454113Z","shell.execute_reply":"2022-05-17T15:31:02.463640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_name = meta_data['label_names']\nlabels_name","metadata":{"id":"JrCnPNqjPZMy","outputId":"3176a5e5-bbd4-4e1b-8b1a-d61c3d45278c","execution":{"iopub.status.busy":"2022-05-17T15:31:02.465907Z","iopub.execute_input":"2022-05-17T15:31:02.466262Z","iopub.status.idle":"2022-05-17T15:31:02.475623Z","shell.execute_reply.started":"2022-05-17T15:31:02.466228Z","shell.execute_reply":"2022-05-17T15:31:02.474247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_dict = {k:v for k,v in zip(one_hot_labels, labels_name)} # dict(zip(labels, one_hot))\nlabel_dict","metadata":{"id":"5CSPwlxRPuVe","outputId":"8420a161-85cc-4e17-af44-e3a34ca5d559","execution":{"iopub.status.busy":"2022-05-17T15:31:02.481743Z","iopub.execute_input":"2022-05-17T15:31:02.481990Z","iopub.status.idle":"2022-05-17T15:31:02.494455Z","shell.execute_reply.started":"2022-05-17T15:31:02.481958Z","shell.execute_reply":"2022-05-17T15:31:02.493564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_dict.get(8)","metadata":{"id":"7bZciNerSABJ","outputId":"af7f1100-70a8-40da-897d-59faf34631d6","execution":{"iopub.status.busy":"2022-05-17T15:31:02.496048Z","iopub.execute_input":"2022-05-17T15:31:02.496338Z","iopub.status.idle":"2022-05-17T15:31:02.506782Z","shell.execute_reply.started":"2022-05-17T15:31:02.496304Z","shell.execute_reply":"2022-05-17T15:31:02.505879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the shape of the dataset \ntrain_data = list(train_dataset)\ntest_data  = list(test_dataset)\nprint(f'Train data shape: {len(train_data)}')\nprint(f'Test data shape:  {len(test_data)}')","metadata":{"id":"a18gWxNsJ6Fy","outputId":"d737ad54-82bd-4f10-87f7-64ac2f09bffa","execution":{"iopub.status.busy":"2022-05-17T15:31:02.508957Z","iopub.execute_input":"2022-05-17T15:31:02.510013Z","iopub.status.idle":"2022-05-17T15:31:18.825067Z","shell.execute_reply.started":"2022-05-17T15:31:02.509978Z","shell.execute_reply":"2022-05-17T15:31:18.823209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the shape of an image in the train data\nimage, label = train_data[300]\nprint(image.shape)\nprint(label)","metadata":{"id":"k9Dr39k3O-pC","outputId":"cf73ac77-1e95-48ef-d475-884fdbecb1fc","execution":{"iopub.status.busy":"2022-05-17T15:31:18.826624Z","iopub.execute_input":"2022-05-17T15:31:18.826895Z","iopub.status.idle":"2022-05-17T15:31:18.832096Z","shell.execute_reply.started":"2022-05-17T15:31:18.826860Z","shell.execute_reply":"2022-05-17T15:31:18.831249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check some random training images \ndef imshow(img, label):\n    img = img / 2 + 0.5     # unnormalize\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0))) # tensor.permute = np.transpose\n    plt.title(''.join(f'{label_dict.get(label)}' )) # show label\n    \n\n# plt.imshow(image.permute(1,2,0))\n# plt.show()\n\n# get some random training images\nimage, label = train_data[20]\n\nimshow(image, label)\n\nplt.show()","metadata":{"id":"p5YXgU39KcWv","outputId":"f6479f94-210a-442c-9cc6-f2f9c060be3f","execution":{"iopub.status.busy":"2022-05-17T15:31:18.833312Z","iopub.execute_input":"2022-05-17T15:31:18.833791Z","iopub.status.idle":"2022-05-17T15:31:19.031908Z","shell.execute_reply.started":"2022-05-17T15:31:18.833753Z","shell.execute_reply":"2022-05-17T15:31:19.031247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create DataLoader","metadata":{"id":"-4gkeIyrTQI-"}},{"cell_type":"code","source":"batch_size = 64         # hyper-parameter \ntrain_loader = torch.utils.data.DataLoader(\n                            dataset = train_dataset, \n                            batch_size = batch_size, \n                            shuffle = True)\n\ntest_loader = torch.utils.data.DataLoader(\n                            dataset = test_dataset, \n                            batch_size = batch_size, \n                            shuffle = True)","metadata":{"id":"4AHWYrRdSDqL","execution":{"iopub.status.busy":"2022-05-17T15:31:19.033073Z","iopub.execute_input":"2022-05-17T15:31:19.035602Z","iopub.status.idle":"2022-05-17T15:31:19.040828Z","shell.execute_reply.started":"2022-05-17T15:31:19.035561Z","shell.execute_reply":"2022-05-17T15:31:19.040097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the shape of the batches \ndd = list(train_loader)\n","metadata":{"id":"7izlqARaTbro","execution":{"iopub.status.busy":"2022-05-17T15:31:19.042126Z","iopub.execute_input":"2022-05-17T15:31:19.042849Z","iopub.status.idle":"2022-05-17T15:31:31.694930Z","shell.execute_reply.started":"2022-05-17T15:31:19.042811Z","shell.execute_reply":"2022-05-17T15:31:31.694210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = dd[0]\nprint(images.shape)\nprint(labels.shape)\n","metadata":{"id":"YUviNHl_H-Vb","outputId":"2b73325f-83a5-44f9-9680-ff3acf7ece12","execution":{"iopub.status.busy":"2022-05-17T15:31:31.696174Z","iopub.execute_input":"2022-05-17T15:31:31.696404Z","iopub.status.idle":"2022-05-17T15:31:31.702923Z","shell.execute_reply.started":"2022-05-17T15:31:31.696372Z","shell.execute_reply":"2022-05-17T15:31:31.702115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining the Networks\n\nIn this section, we'll build three neural network models: \n* Multi-Layer Perceptron (MLP)\n* Convolutional Neural Network (CNN)\n* ResNet9","metadata":{"id":"1hOkZBLTVszK"}},{"cell_type":"markdown","source":"## MLP","metadata":{"id":"PfLgGkQ2j2PS"}},{"cell_type":"code","source":"class MLP(torch.nn.Module):\n  def __init__(self, input_size, n_hidden_units, n_classes):\n    super(MLP, self).__init__()\n\n    h1, h2, h3 = n_hidden_units\n\n    # Add Linear layers\n    self.fc1 = nn.Linear(input_size,h1)\n    self.fc2 = nn.Linear(h1, h2)\n    self.fc3 = nn.Linear(h2, h3)\n    self.fc4 = nn.Linear(h3, n_classes) # 10 classes\n    self.dropout = nn.Dropout(0.25)\n\n  def forward(self, x):\n    x = x.view(-1, input_size) # Flatten out the input layer\n    x = F.relu(self.fc1(x))\n    x = F.relu(self.fc2(x))\n    x = F.relu(self.fc3(x))\n    x = self.dropout(x)\n    x = self.fc4(x)\n    return x\n\n","metadata":{"id":"YupWjkpfTeg5","execution":{"iopub.status.busy":"2022-05-17T15:31:31.704452Z","iopub.execute_input":"2022-05-17T15:31:31.704935Z","iopub.status.idle":"2022-05-17T15:31:31.714088Z","shell.execute_reply.started":"2022-05-17T15:31:31.704899Z","shell.execute_reply":"2022-05-17T15:31:31.713245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize MLP model\ninput_size = IMAGE_HEIGHT*IMAGE_WIDTH*COLOR_CHANNELS\nn_hidden_units = [512, 256, 128]\n\nmlp_model = MLP(input_size, n_hidden_units, N_CLASSES)","metadata":{"id":"sMEvqMATyQzP","execution":{"iopub.status.busy":"2022-05-17T15:31:31.716581Z","iopub.execute_input":"2022-05-17T15:31:31.716854Z","iopub.status.idle":"2022-05-17T15:31:31.739232Z","shell.execute_reply.started":"2022-05-17T15:31:31.716769Z","shell.execute_reply":"2022-05-17T15:31:31.738514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check the shape of MLP model\noutput = mlp_model(images)\noutput.shape","metadata":{"id":"EGRwm4W_yR1K","outputId":"b0a8b1ce-e8ba-404b-d58f-ff9b50fbd445","execution":{"iopub.status.busy":"2022-05-17T15:31:31.740553Z","iopub.execute_input":"2022-05-17T15:31:31.741022Z","iopub.status.idle":"2022-05-17T15:31:31.798260Z","shell.execute_reply.started":"2022-05-17T15:31:31.740974Z","shell.execute_reply":"2022-05-17T15:31:31.797348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN","metadata":{"id":"5RXl8cVnj35k"}},{"cell_type":"code","source":"def conv3x3(in_channels, out_channels, stride=1):\n  \"\"\"3x3 convolution with padding\"\"\"\n  return nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:31.799580Z","iopub.execute_input":"2022-05-17T15:31:31.799808Z","iopub.status.idle":"2022-05-17T15:31:31.805581Z","shell.execute_reply.started":"2022-05-17T15:31:31.799776Z","shell.execute_reply":"2022-05-17T15:31:31.804730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CNN(torch.nn.Module):\n  def __init__(self, in_channels, n_classes):\n    super(CNN, self).__init__()\n    # Conv layers\n    self.conv =conv3x3(in_channels=in_channels, out_channels=64)\n    self.bn = nn.BatchNorm2d(64)\n    self.relu = nn.ReLU(inplace=True)\n\n    #self.conv0 =conv3x3(in_channels=in_channels, out_channels=64)\n    #self.bn0 = nn.BatchNorm2d(64)\n\n    # Residual block\n    self.conv1 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1)\n    self.bn1 = nn.BatchNorm2d(64)\n    self.relu1 = nn.ReLU(inplace=True)\n\n    self.conv2 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=1)\n    self.bn2 = nn.BatchNorm2d(64)\n    self.relu2 = nn.ReLU(inplace=True)\n\n    # Pooling\n    self.avg_pool = nn.AvgPool2d(2)\n\n    # Linear layers\n    self.fc1 = nn.Linear(16384, 64 )\n    self.fc2 = nn.Linear(64, n_classes)\n\n  def forward(self, x):\n    \n    #print(identity.shape)\n    out = self.conv(x)\n    out = self.bn(out)\n    out = self.relu(out)\n\n    #out = self.conv0(x)\n    #out = self.bn0(out)\n    #out = self.relu(out)\n\n    out = self.conv1(out)\n    out = self.bn1(out)\n    out = self.relu1(out)\n\n    out = self.conv2(out)\n    out = self.bn2(out)\n    out = self.relu2(out)  # skip connection + out\n    #print(out.shape)\n\n    out = self.avg_pool(out)\n\n    out = out.view(out.size(0), -1) # Flatten all dimension except batch\n    #print(out.shape)\n\n    out = self.fc1(out)\n    out = F.relu(out)\n\n    out = self.fc2(out)\n    #out = F.log_softmax(out, dim=-1) # if using cross entropy, dont need to use log softmax since it's already implemented in cross entropy\n\n    return out","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:31.807537Z","iopub.execute_input":"2022-05-17T15:31:31.807973Z","iopub.status.idle":"2022-05-17T15:31:31.821194Z","shell.execute_reply.started":"2022-05-17T15:31:31.807930Z","shell.execute_reply":"2022-05-17T15:31:31.820484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model = CNN(in_channels=3, n_classes=10)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:31.822685Z","iopub.execute_input":"2022-05-17T15:31:31.823189Z","iopub.status.idle":"2022-05-17T15:31:31.842012Z","shell.execute_reply.started":"2022-05-17T15:31:31.823152Z","shell.execute_reply":"2022-05-17T15:31:31.841335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = dd[0]\nprint(images.shape)\nprint(labels.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:31.844513Z","iopub.execute_input":"2022-05-17T15:31:31.844909Z","iopub.status.idle":"2022-05-17T15:31:31.852214Z","shell.execute_reply.started":"2022-05-17T15:31:31.844869Z","shell.execute_reply":"2022-05-17T15:31:31.851310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the output shape of the model\noutput = cnn_model(images)\noutput.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:31.853616Z","iopub.execute_input":"2022-05-17T15:31:31.853962Z","iopub.status.idle":"2022-05-17T15:31:32.128553Z","shell.execute_reply.started":"2022-05-17T15:31:31.853914Z","shell.execute_reply":"2022-05-17T15:31:32.127871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ResNet9","metadata":{"id":"RmNZUmhzPOQi"}},{"cell_type":"code","source":"class BasicResidualBlock(nn.Module):\n  def __init__(self):\n    super(BasicResidualBlock, self).__init__()\n    self.conv1 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3, stride=1, padding=1)\n    self.relu1 = nn.ReLU()\n    self.conv2 = nn.Conv2d(in_channels=3, out_channels=3, kernel_size=3, stride=1, padding=1)\n    self.relu2 = nn.ReLU()\n\n  def forward(self, x):\n    identity = x\n    out = self.conv1(x)\n    out = self.relu1(out)\n    out = self.conv2(out)\n    out = self.relu2(out)\n    return self.relu2(out) + identity # ReLU can be applied before or after adding the identity/input","metadata":{"id":"Ikce9AVPNkpM","execution":{"iopub.status.busy":"2022-05-17T15:31:32.129823Z","iopub.execute_input":"2022-05-17T15:31:32.130226Z","iopub.status.idle":"2022-05-17T15:31:32.137709Z","shell.execute_reply.started":"2022-05-17T15:31:32.130189Z","shell.execute_reply":"2022-05-17T15:31:32.136850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking the output size of the BasicResidualBlock\nsimple_resblock = BasicResidualBlock()\nimages, labels = dd[0]\nprint(images.shape)\nprint(labels.shape)","metadata":{"id":"GGXeu9tKOoKG","outputId":"28d54cf0-10da-4c8b-e1d3-f1c9661f5455","execution":{"iopub.status.busy":"2022-05-17T15:31:32.138914Z","iopub.execute_input":"2022-05-17T15:31:32.139251Z","iopub.status.idle":"2022-05-17T15:31:32.151179Z","shell.execute_reply.started":"2022-05-17T15:31:32.139215Z","shell.execute_reply":"2022-05-17T15:31:32.149941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out = simple_resblock(images)\nout.shape","metadata":{"id":"99zWUOTkOytJ","outputId":"8c491987-49b2-4209-f83d-36db52277fe2","execution":{"iopub.status.busy":"2022-05-17T15:31:32.157125Z","iopub.execute_input":"2022-05-17T15:31:32.158685Z","iopub.status.idle":"2022-05-17T15:31:32.173876Z","shell.execute_reply.started":"2022-05-17T15:31:32.158658Z","shell.execute_reply":"2022-05-17T15:31:32.173230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def conv_block(in_channels, out_channels, pool=False):\n  layers = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(inplace=True)]\n  if pool: \n    layers.append(nn.MaxPool2d(2))\n  return nn.Sequential(*layers)","metadata":{"id":"YoKJSnT1Pbe4","execution":{"iopub.status.busy":"2022-05-17T15:31:32.174983Z","iopub.execute_input":"2022-05-17T15:31:32.175447Z","iopub.status.idle":"2022-05-17T15:31:32.181483Z","shell.execute_reply.started":"2022-05-17T15:31:32.175397Z","shell.execute_reply":"2022-05-17T15:31:32.180831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResNet9(nn.Module):\n  def __init__(self, in_channels, n_classes):\n    super(ResNet9, self).__init__()\n\n    self.conv1 = conv_block(in_channels, 64)\n    self.conv2 = conv_block(64, 128, pool=True)\n    self.res1  = nn.Sequential(conv_block(128,128), conv_block(128,128))\n\n    self.conv3 = conv_block(128, 256, pool=True)\n    self.conv4 = conv_block(256, 512, pool=True)\n    self.res2  = nn.Sequential(conv_block(512,512), conv_block(512,512))\n\n    self.classifier = nn.Sequential(nn.MaxPool2d(4),\n                                    nn.Flatten(),\n                                    nn.Linear(512, 128),\n                                    nn.Linear(128, n_classes))\n    \n  def forward(self, x):\n    out = self.conv1(x)\n    out = self.conv2(out)\n    out = self.res1(out) + out # skip connection\n    out = self.conv3(out)\n    out = self.conv4(out)\n    out = self.res2(out) + out # skip connection\n    out = self.classifier(out)\n    return out","metadata":{"id":"aWyCm_2BQPox","execution":{"iopub.status.busy":"2022-05-17T15:31:32.184763Z","iopub.execute_input":"2022-05-17T15:31:32.184957Z","iopub.status.idle":"2022-05-17T15:31:32.194815Z","shell.execute_reply.started":"2022-05-17T15:31:32.184935Z","shell.execute_reply":"2022-05-17T15:31:32.194104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet_model = ResNet9(3,10)\nresnet_model","metadata":{"id":"7YPMQOFASN7j","outputId":"6ff3d111-2f12-4ba4-f187-bec0ee0799fa","execution":{"iopub.status.busy":"2022-05-17T15:31:32.195792Z","iopub.execute_input":"2022-05-17T15:31:32.198583Z","iopub.status.idle":"2022-05-17T15:31:32.263653Z","shell.execute_reply.started":"2022-05-17T15:31:32.198542Z","shell.execute_reply":"2022-05-17T15:31:32.262953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = dd[0]\nprint(images.shape)\nprint(labels.shape)","metadata":{"id":"HZ4vJ2qcTT1W","outputId":"33d09830-897e-48a8-9f53-889fc0d23fb4","execution":{"iopub.status.busy":"2022-05-17T15:31:32.264736Z","iopub.execute_input":"2022-05-17T15:31:32.265291Z","iopub.status.idle":"2022-05-17T15:31:32.271039Z","shell.execute_reply.started":"2022-05-17T15:31:32.265249Z","shell.execute_reply":"2022-05-17T15:31:32.270293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out = resnet_model(images)\nout.shape","metadata":{"id":"zsYMcW9DSxrH","outputId":"e5a2f995-1f7c-4c71-b75a-cff41edb08ff","execution":{"iopub.status.busy":"2022-05-17T15:31:32.272264Z","iopub.execute_input":"2022-05-17T15:31:32.273037Z","iopub.status.idle":"2022-05-17T15:31:33.010912Z","shell.execute_reply.started":"2022-05-17T15:31:32.272999Z","shell.execute_reply":"2022-05-17T15:31:33.010216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper functions","metadata":{"id":"7IMf8lYJk8ao"}},{"cell_type":"code","source":"def prediction(data_loader, model, criterion, cuda=None):\n  correct = 0\n  total = 0\n  losses = 0\n\n  for i, (images, labels) in enumerate(data_loader):\n    if cuda is not None:\n      # switch tensor type to GPU\n      images = images.cuda()\n      labels = labels.cuda()\n    \n    # Flatten the images\n    #images = images.view(-1, input_size)\n    \n    outputs = model(images)\n    \n    loss = criterion(outputs, labels)\n  \n    _, predictions = torch.max(outputs, dim=1)\n  \n    correct += torch.sum(labels == predictions).item()\n    total += labels.shape[0]\n    \n    losses += loss.data.item()\n    \n  return losses/len(list(data_loader)), 1 - correct/total # we need to normalize loss with respect to the number of batches ","metadata":{"id":"375lQSWOZNv6","execution":{"iopub.status.busy":"2022-05-17T15:31:33.012130Z","iopub.execute_input":"2022-05-17T15:31:33.012600Z","iopub.status.idle":"2022-05-17T15:31:33.020960Z","shell.execute_reply.started":"2022-05-17T15:31:33.012556Z","shell.execute_reply":"2022-05-17T15:31:33.020147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define a Loss function and optimizer","metadata":{"id":"pVX9BJqdxqZT"}},{"cell_type":"markdown","source":"* **Learning rate scheduling**: Learning rate scheduler -> change the learning rate after every batch of training. \"One Cycle Learning Rate Policy\" -> starting with a low learning rate, gradually increasing it batch-by-batch to a high learning rate for about 30% epochs, then gradually decreasing it to a very low value for the remaining epochs.\n* **Weight decay**: Regularization technique to prevent the weights from becoming too large by adding an additional term to the loss function\n* **Gradient clipping**: it's helpful to limit the values of gradients to a small range to prevent undesirable changes in parameters due to large gradient values. ","metadata":{"id":"TuF96yATiWir"}},{"cell_type":"markdown","source":"## Fit one cycle","metadata":{"id":"gUgZt1Ks3ih0"}},{"cell_type":"code","source":"def get_lr(optimizer):\n  for param_group in optimizer.param_groups:\n    return param_group['lr']\n","metadata":{"id":"Bw5J3ttgrBKO","execution":{"iopub.status.busy":"2022-05-17T15:31:33.021966Z","iopub.execute_input":"2022-05-17T15:31:33.023050Z","iopub.status.idle":"2022-05-17T15:31:33.030083Z","shell.execute_reply.started":"2022-05-17T15:31:33.023011Z","shell.execute_reply":"2022-05-17T15:31:33.029478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define a fit_one_cycle function to incorporate learning rate scheduling, weight decay and gradient clipping\ndef fit_one_cycle(epochs, max_lr, model, train_loader, val_loader, criterion,\n                  weight_decay=0, grad_clip=None, opt_func=torch.optim.SGD, cuda=None):\n  torch.cuda.empty_cache() # release all unoccupied cached memory\n  history = []\n\n  # Set up custom optimizer with weight decay\n  optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)\n  # Set up one-cycle learning rate scheduler\n  sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs,\n                                              steps_per_epoch=len(train_loader))\n  \n  if cuda is not None:\n    model.cuda() # send model to device\n\n  train_losses = []\n  test_losses = []\n\n  train_error_rates = []\n  test_error_rates = []\n\n  # Loop through all epochs\n  for epoch in range(epochs):\n    train_loss = 0\n    n_iter = 0\n    total = 0\n    correct = 0\n    lrs = []\n\n    # Training phase\n    for i, (images, labels) in enumerate(train_loader):\n      optimizer.zero_grad() # zero gradients out before computing others\n\n      if cuda is not None:\n        images = images.cuda()\n        labels = labels.cuda()\n\n      #images = images.view(-1, input_size) # Flatten input image\n      # dont need in this case since it's CNN model\n      outputs = model(images) \n\n      # compute the train_error rates\n      # torch.max(input, dim) ->\n      # returns the maximum value of all elements in the input tensor\n      _, predictions = torch.max(outputs, 1) \n      correct += torch.sum(labels == predictions).item()\n      total += labels.shape[0]\n\n      # compute loss\n      loss = criterion(outputs, labels)\n      # compute gradients\n      loss.backward()\n\n      # Gradient clipping:\n      if grad_clip is not None:\n        nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n\n      # update weights/parameters\n      optimizer.step()\n\n      # record and update learning rate\n      lrs.append(get_lr(optimizer))\n      sched.step()\n\n      train_loss += loss.detach().item()\n\n      n_iter += 1\n    \n    train_error_rate = 1 - correct/total\n\n    # Validation phase\n    with torch.no_grad():\n      test_loss, test_error_rate = prediction(val_loader, model, criterion, cuda)\n\n    train_error_rates.append(train_error_rate)\n    test_error_rates.append(test_error_rate)\n    train_losses.append(train_loss/n_iter)\n    test_losses.append(test_loss)\n    results ={'train_loss': train_loss/n_iter,'val_loss': test_loss, 'val_acc': (1-train_error_rate)*100}\n    results['lrs'] = lrs\n    history.append(results)\n\n    if epoch%1 == 0:\n      print('Epoch: {}/{}, last_lr: {:.5f}, train_loss: {:.4f}, val_loss: {:.4f}, val_accuracy: {:.1f}%'.format(epoch+1, epochs, results['lrs'][-1], train_loss/n_iter, test_loss, (1-train_error_rate)*100))\n\n  return history\n    \n    ","metadata":{"id":"kR4YKMbfsq9q","execution":{"iopub.status.busy":"2022-05-17T15:31:33.031671Z","iopub.execute_input":"2022-05-17T15:31:33.032192Z","iopub.status.idle":"2022-05-17T15:31:33.048263Z","shell.execute_reply.started":"2022-05-17T15:31:33.032151Z","shell.execute_reply":"2022-05-17T15:31:33.047514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the networks","metadata":{"id":"DICmoSrRzd55"}},{"cell_type":"markdown","source":"## Train with lr_scheduler\n","metadata":{"id":"w7evzBOQz509"}},{"cell_type":"markdown","source":"### Optimizer and loss function","metadata":{}},{"cell_type":"code","source":"epochs = 20\nmax_lr = 0.01\ngrad_clip = 0.1\nweight_decay = 1e-4\nopt_func = torch.optim.Adam\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:33.049717Z","iopub.execute_input":"2022-05-17T15:31:33.050372Z","iopub.status.idle":"2022-05-17T15:31:33.058681Z","shell.execute_reply.started":"2022-05-17T15:31:33.050334Z","shell.execute_reply":"2022-05-17T15:31:33.058018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### MLP","metadata":{}},{"cell_type":"code","source":"history_mlp = fit_one_cycle(epochs, max_lr, mlp_model, train_loader, test_loader, criterion, weight_decay, opt_func=opt_func, cuda=cuda  )","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:31:33.059658Z","iopub.execute_input":"2022-05-17T15:31:33.061711Z","iopub.status.idle":"2022-05-17T15:37:52.933603Z","shell.execute_reply.started":"2022-05-17T15:31:33.061609Z","shell.execute_reply":"2022-05-17T15:37:52.932817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save model\nPATH_MLP = './mlp_cifar_net.pth'\ntorch.save(mlp_model.state_dict(), PATH_MLP)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:37:52.935126Z","iopub.execute_input":"2022-05-17T15:37:52.935591Z","iopub.status.idle":"2022-05-17T15:37:52.955242Z","shell.execute_reply.started":"2022-05-17T15:37:52.935539Z","shell.execute_reply":"2022-05-17T15:37:52.954571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### CNN","metadata":{}},{"cell_type":"code","source":"history_cnn = fit_one_cycle(epochs, max_lr, cnn_model, train_loader, test_loader, criterion, weight_decay, opt_func=opt_func, cuda=cuda  )","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:37:52.956422Z","iopub.execute_input":"2022-05-17T15:37:52.956744Z","iopub.status.idle":"2022-05-17T15:46:39.356527Z","shell.execute_reply.started":"2022-05-17T15:37:52.956709Z","shell.execute_reply":"2022-05-17T15:46:39.355827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save model\nPATH_CNN = './cnn_cifar_net.pth'\ntorch.save(cnn_model.state_dict(), PATH_CNN)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T15:46:39.357687Z","iopub.execute_input":"2022-05-17T15:46:39.358752Z","iopub.status.idle":"2022-05-17T15:46:39.378080Z","shell.execute_reply.started":"2022-05-17T15:46:39.358713Z","shell.execute_reply":"2022-05-17T15:46:39.377394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ResNet9","metadata":{}},{"cell_type":"code","source":"history_resnet = fit_one_cycle(epochs, max_lr, resnet_model, train_loader, test_loader, criterion, weight_decay, opt_func=opt_func, cuda=cuda  )","metadata":{"id":"OVv2nykj0Dft","outputId":"f217f397-5be0-41fb-d021-fd659e621eda","execution":{"iopub.status.busy":"2022-05-17T15:46:39.379362Z","iopub.execute_input":"2022-05-17T15:46:39.379651Z","iopub.status.idle":"2022-05-17T15:58:45.263455Z","shell.execute_reply.started":"2022-05-17T15:46:39.379617Z","shell.execute_reply":"2022-05-17T15:58:45.262633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_RESNET = './rescnn_cifar_lr_net.pth'\ntorch.save(resnet_model.state_dict(), PATH_RESNET)","metadata":{"id":"mlJFggKN96nk","execution":{"iopub.status.busy":"2022-05-17T15:58:45.264872Z","iopub.execute_input":"2022-05-17T15:58:45.265138Z","iopub.status.idle":"2022-05-17T15:58:45.320089Z","shell.execute_reply.started":"2022-05-17T15:58:45.265100Z","shell.execute_reply":"2022-05-17T15:58:45.319287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluate the network","metadata":{"id":"zFO2kHnBKqgf"}},{"cell_type":"code","source":"def plot_accuracies(history):\n    accuracies = [x['val_acc'] for x in history]\n    \n    plt.plot(accuracies, '-x')\n    plt.xlabel('epoch')\n    plt.ylabel('accuracy')\n    plt.title('Accuracy vs. No. of epochs');","metadata":{"id":"244mIwc83EnX","execution":{"iopub.status.busy":"2022-05-17T15:58:45.321646Z","iopub.execute_input":"2022-05-17T15:58:45.321915Z","iopub.status.idle":"2022-05-17T15:58:45.329738Z","shell.execute_reply.started":"2022-05-17T15:58:45.321875Z","shell.execute_reply":"2022-05-17T15:58:45.328982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_losses(history):\n  #plt.figure(figsize=(10, 8))\n  #plt.subplot(1,2,1)\n  train_losses = [x.get('train_loss') for x in history]\n  test_losses = [x['val_loss'] for x in history]\n  plt.plot(train_losses, label='train', marker='o', alpha=0.7)\n  plt.plot(test_losses, label='test', marker='o', alpha=0.7)\n  plt.xlabel(\"Epoch\")\n  plt.ylabel(\"Loss\")\n  plt.grid(True)\n  plt.legend()\n  plt.title('Model Loss')","metadata":{"id":"WicCyw6qBRyu","execution":{"iopub.status.busy":"2022-05-17T15:58:45.331318Z","iopub.execute_input":"2022-05-17T15:58:45.331644Z","iopub.status.idle":"2022-05-17T15:58:45.338822Z","shell.execute_reply.started":"2022-05-17T15:58:45.331608Z","shell.execute_reply":"2022-05-17T15:58:45.338086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.subplot(1,3,1)\nplot_losses(history_mlp)\nplt.title('MLP')\nplt.subplot(1,3,2)\nplot_losses(history_cnn)\nplt.title('CNN')\nplt.subplot(1,3,3)\nplot_losses(history_resnet)\nplt.title('ResNet9')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T16:00:46.060140Z","iopub.execute_input":"2022-05-17T16:00:46.060486Z","iopub.status.idle":"2022-05-17T16:00:46.686370Z","shell.execute_reply.started":"2022-05-17T16:00:46.060445Z","shell.execute_reply":"2022-05-17T16:00:46.685687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nplt.subplot(1,3,1)\nplot_accuracies(history_mlp)\nplt.title('MLP')\nplt.subplot(1,3,2)\nplot_accuracies(history_cnn)\nplt.title('CNN')\nplt.subplot(1,3,3)\nplot_accuracies(history_resnet)\nplt.title('ResNet9')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T16:00:54.317605Z","iopub.execute_input":"2022-05-17T16:00:54.317857Z","iopub.status.idle":"2022-05-17T16:00:54.668235Z","shell.execute_reply.started":"2022-05-17T16:00:54.317828Z","shell.execute_reply":"2022-05-17T16:00:54.667578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualize the prediction","metadata":{"id":"r_rMOEkzFuWx"}},{"cell_type":"code","source":"dataiter = iter(test_loader)\nimages, labels = dataiter.next()","metadata":{"id":"pflb2vlzGC5B","execution":{"iopub.status.busy":"2022-05-17T15:58:46.099314Z","iopub.execute_input":"2022-05-17T15:58:46.100270Z","iopub.status.idle":"2022-05-17T15:58:46.124513Z","shell.execute_reply.started":"2022-05-17T15:58:46.100230Z","shell.execute_reply":"2022-05-17T15:58:46.123905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images.shape","metadata":{"id":"rOk1nCVqGD27","outputId":"de4bdae4-ec66-4d8b-e5a1-c6e4b4da2e19","execution":{"iopub.status.busy":"2022-05-17T15:58:46.125629Z","iopub.execute_input":"2022-05-17T15:58:46.125933Z","iopub.status.idle":"2022-05-17T15:58:46.131356Z","shell.execute_reply.started":"2022-05-17T15:58:46.125891Z","shell.execute_reply":"2022-05-17T15:58:46.130641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape","metadata":{"id":"2bS1jgMmGJbt","outputId":"ed482c6d-abdb-4e73-fbd9-9f081a17511e","execution":{"iopub.status.busy":"2022-05-17T15:58:46.132600Z","iopub.execute_input":"2022-05-17T15:58:46.133358Z","iopub.status.idle":"2022-05-17T15:58:46.141837Z","shell.execute_reply.started":"2022-05-17T15:58:46.133321Z","shell.execute_reply":"2022-05-17T15:58:46.140842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classes = ('plane', 'car', 'bird', 'cat',\n           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')","metadata":{"id":"FaPwOHuIGjc-","execution":{"iopub.status.busy":"2022-05-17T15:58:46.143036Z","iopub.execute_input":"2022-05-17T15:58:46.143507Z","iopub.status.idle":"2022-05-17T15:58:46.149285Z","shell.execute_reply.started":"2022-05-17T15:58:46.143468Z","shell.execute_reply":"2022-05-17T15:58:46.148641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataiter = iter(test_loader)\nimages, labels = dataiter.next()\ndef imshow(img):\n    img = img / 2 + 0.5     # unnormalize\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0))) # tensor.permute = np.transpose\n# print images\nimshow(torchvision.utils.make_grid(images[:5]))\nprint('GroundTruth: ', ' '.join(f'{classes[labels[j]]:5s}' for j in range(5)))","metadata":{"id":"6Oo629ZPFuBv","outputId":"1f4253d9-a7a5-4b4b-a7d0-ec20b6aa2be6","execution":{"iopub.status.busy":"2022-05-17T15:58:46.150686Z","iopub.execute_input":"2022-05-17T15:58:46.151001Z","iopub.status.idle":"2022-05-17T15:58:46.353825Z","shell.execute_reply.started":"2022-05-17T15:58:46.150897Z","shell.execute_reply":"2022-05-17T15:58:46.353169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet = ResNet9(in_channels=3, n_classes=10)\nresnet.load_state_dict(torch.load(PATH_RESNET))","metadata":{"id":"CkR3iKCAYRQw","execution":{"iopub.status.busy":"2022-05-17T15:58:46.355178Z","iopub.execute_input":"2022-05-17T15:58:46.355577Z","iopub.status.idle":"2022-05-17T15:58:46.440278Z","shell.execute_reply.started":"2022-05-17T15:58:46.355539Z","shell.execute_reply":"2022-05-17T15:58:46.439420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"outputs = resnet(images)","metadata":{"id":"sZGz-j2IG1lY","execution":{"iopub.status.busy":"2022-05-17T15:58:46.441794Z","iopub.execute_input":"2022-05-17T15:58:46.442078Z","iopub.status.idle":"2022-05-17T15:59:01.104308Z","shell.execute_reply.started":"2022-05-17T15:58:46.442042Z","shell.execute_reply":"2022-05-17T15:59:01.103587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, predicted = torch.max(outputs, 1)\n\nprint('Predicted: ', ' '.join(f'{classes[predicted[j]]:5s}'\n                              for j in range(5)))","metadata":{"id":"BkeaPeFIG8rk","outputId":"53018f9c-98e8-4c19-dfa4-317b8d1f1198","execution":{"iopub.status.busy":"2022-05-17T15:59:01.105657Z","iopub.execute_input":"2022-05-17T15:59:01.105903Z","iopub.status.idle":"2022-05-17T15:59:01.112780Z","shell.execute_reply.started":"2022-05-17T15:59:01.105869Z","shell.execute_reply":"2022-05-17T15:59:01.112098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"iSG04xFFEFsP"},"execution_count":null,"outputs":[]}]}