{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-12T12:20:13.956325Z","iopub.execute_input":"2023-10-12T12:20:13.956948Z","iopub.status.idle":"2023-10-12T12:20:13.969611Z","shell.execute_reply.started":"2023-10-12T12:20:13.956916Z","shell.execute_reply":"2023-10-12T12:20:13.968654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Note\n\n* reference:\n    * from scratch\n        - https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py\n        - https://github.com/KaimingHe/deep-residual-networks\n    * transfer learning :\n        - https://cs231n.github.io/transfer-learning/\n        - https://pytorch.org/hub/pytorch_vision_resnet/\n        \n## Video walkthrough:\n- [link](https://youtube.com/shorts/OrjzD_dnCjI?feature=share)\n\n## Contents:\n* [Import packages](#1)\n    * [Seed everything](#11)\n    * [Set deterministic](#12)\n* [Preprocess image](#2)\n    * [Unzip](#21)    \n    * [Show image examples](#22)\n    * [Check image sizes](#23)    \n    * [Augmentation while loading](#24)\n    * [Check augmented images](#25)\n* [Build a ResNet-101 model from scratch](#3)\n    * [Define model](#31)\n    * [Choose loss function, optimizer](#33)\n    * [Train model](#34)\n    * [Visualize the learning curve](#35)\n    * [Make Predictions on test set](#36)\n* [Build a model using ResNet-152 pretrained weights](#4)\n    * [Load pre-trained ResNet-152](#41)\n    * [Freeze early layers](#42)\n    * [Modify output layer](#43)\n    * [Train model](#44)\n    * [Visualize the learning curve](#45)\n    * [Make Predictions on test set](#46)","metadata":{}},{"cell_type":"markdown","source":"## 1. Import packages <a class=\"anchor\" id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nimport time\nimport multiprocessing\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nimport torch.nn.functional as F\nimport torchvision.models as models\n\n# from torchviz import make_dot\nimport random\n\nfrom dataclasses import make_dataclass\n\nfrom typing import Any, Callable, List, Optional, Tuple\nfrom torch import Tensor","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:22:42.721337Z","iopub.execute_input":"2023-10-12T12:22:42.721653Z","iopub.status.idle":"2023-10-12T12:22:46.802524Z","shell.execute_reply.started":"2023-10-12T12:22:42.721630Z","shell.execute_reply":"2023-10-12T12:22:46.801614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.1 Seed everything  <a class=\"anchor\" id=\"11\"></a>","metadata":{}},{"cell_type":"code","source":"SEED = 1\nos.environ[\"PL_GLOBAL_SEED\"] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:22:46.804169Z","iopub.execute_input":"2023-10-12T12:22:46.804717Z","iopub.status.idle":"2023-10-12T12:22:46.815002Z","shell.execute_reply.started":"2023-10-12T12:22:46.804683Z","shell.execute_reply":"2023-10-12T12:22:46.814163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Set cuDNN and PyTorch to be deterministic <a class=\"anchor\" id=\"12\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:05:19.156546Z","iopub.execute_input":"2023-09-15T11:05:19.156906Z","iopub.status.idle":"2023-09-15T11:05:19.161724Z","shell.execute_reply.started":"2023-09-15T11:05:19.156877Z","shell.execute_reply":"2023-09-15T11:05:19.16052Z"}}},{"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n# torch.set_deterministic(True)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:22:46.816464Z","iopub.execute_input":"2023-10-12T12:22:46.816807Z","iopub.status.idle":"2023-10-12T12:22:46.898739Z","shell.execute_reply.started":"2023-10-12T12:22:46.816777Z","shell.execute_reply":"2023-10-12T12:22:46.897645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Settings","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nLEARNING_RATE = 0.0001\nBATCH_SIZE = 64\nNUM_EPOCHS = 40\n\nNUM_CLASSES = 2\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:22:46.902464Z","iopub.execute_input":"2023-10-12T12:22:46.902799Z","iopub.status.idle":"2023-10-12T12:22:46.912090Z","shell.execute_reply.started":"2023-10-12T12:22:46.902773Z","shell.execute_reply":"2023-10-12T12:22:46.911157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Preprocess image <a class=\"anchor\" id=\"2\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Unzip <a class=\"anchor\" id=\"21\"></a>","metadata":{}},{"cell_type":"code","source":"base_dir = '../input/dogs-vs-cats-redux-kernels-edition'\ntrain_dir = '../data/train'\ntest_dir = '../data/test'\n\nwith zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:\n    train_zip.extractall('../data')\n    \nwith zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:\n    test_zip.extractall('../data')\n    \ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\nlen(train_list), len(test_list)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:22:46.913752Z","iopub.execute_input":"2023-10-12T12:22:46.914349Z","iopub.status.idle":"2023-10-12T12:23:02.970537Z","shell.execute_reply.started":"2023-10-12T12:22:46.914313Z","shell.execute_reply":"2023-10-12T12:23:02.969642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Show training image examples <a class=\"anchor\" id=\"22\"></a>","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure(figsize = (15,6))\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = Image.open(train_list[idx])\n    img_path = train_list[idx]\n    label = img_path.split('/')[-1].split('.')[0]\n    plt.imshow(img)\n    plt.title(label, size=15)\n    i+=1\n    \nplt.tight_layout()\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:02.971950Z","iopub.execute_input":"2023-10-12T12:23:02.973199Z","iopub.status.idle":"2023-10-12T12:23:05.840993Z","shell.execute_reply.started":"2023-10-12T12:23:02.973164Z","shell.execute_reply":"2023-10-12T12:23:05.840056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Check the original sizes of the training images and channels <a class=\"anchor\" id=\"23\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:14:14.711891Z","iopub.execute_input":"2023-07-02T22:14:14.712268Z","iopub.status.idle":"2023-07-02T22:14:14.717015Z","shell.execute_reply.started":"2023-07-02T22:14:14.712238Z","shell.execute_reply":"2023-07-02T22:14:14.715667Z"}}},{"cell_type":"code","source":"h_list = []\nw_list = []\nband_set = set()\nfor train_img in train_list:\n    img = Image.open(train_img)\n    w, h = img.size\n    h_list.append(h)\n    w_list.append(w)\n    band_set.add(img.getbands())\n    \nw_med, h_med = np.percentile(w_list, q = 80), np.percentile(h_list, q = 80)\nprint('80 percentile width of all the training images = ', w_med, \n      '\\n80 percentile height of all the training images =', h_med)\nprint('channels of the training images:', band_set)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:05.842011Z","iopub.execute_input":"2023-10-12T12:23:05.842348Z","iopub.status.idle":"2023-10-12T12:23:08.155752Z","shell.execute_reply.started":"2023-10-12T12:23:05.842315Z","shell.execute_reply":"2023-10-12T12:23:08.154796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Image augmentation during data loading <a class=\"anchor\" id=\"24\"></a>","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.manual_seed(SEED)\nif device =='cuda':\n    torch.cuda.manual_seed_all(SEED)\nprint(\"device:\", device)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:08.157056Z","iopub.execute_input":"2023-10-12T12:23:08.157413Z","iopub.status.idle":"2023-10-12T12:23:08.166966Z","shell.execute_reply.started":"2023-10-12T12:23:08.157376Z","shell.execute_reply":"2023-10-12T12:23:08.165997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    'train': transforms.Compose([\n        transforms.RandomResizedCrop([512,512]),\n        transforms.RandomHorizontalFlip(),\n        transforms.Resize([256,256]),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize([256,256]),\n        transforms.ToTensor()\n    ])\n}","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:08.170505Z","iopub.execute_input":"2023-10-12T12:23:08.171261Z","iopub.status.idle":"2023-10-12T12:23:08.222390Z","shell.execute_reply.started":"2023-10-12T12:23:08.171227Z","shell.execute_reply":"2023-10-12T12:23:08.221228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        if self.transform is not None:\n            img = self.transform(img)\n        label = img_path.split('/')[-1].split('.')[0]\n        if label == 'dog':\n            label = 1\n        elif label == 'cat':\n            label = 0\n            \n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:08.224047Z","iopub.execute_input":"2023-10-12T12:23:08.224487Z","iopub.status.idle":"2023-10-12T12:23:08.235185Z","shell.execute_reply.started":"2023-10-12T12:23:08.224453Z","shell.execute_reply":"2023-10-12T12:23:08.234084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trlist, valist = train_test_split(train_list, test_size=0.1)\n\ntr_ds = dataset(trlist, transform=data_transforms['train'])\nva_ds = dataset(valist, transform=data_transforms['test'])\nte_ds = dataset(test_list, transform=data_transforms['test'])\n\n# tr_ds = Subset(tr_ds, torch.arange(1000))\n# va_ds = Subset(va_ds, torch.arange(100))\n\nnum_workers = multiprocessing.cpu_count()\ntr_dl = DataLoader(tr_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers = num_workers, pin_memory = True)\nva_dl = DataLoader(va_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:08.236659Z","iopub.execute_input":"2023-10-12T12:23:08.237060Z","iopub.status.idle":"2023-10-12T12:23:08.258256Z","shell.execute_reply.started":"2023-10-12T12:23:08.237025Z","shell.execute_reply":"2023-10-12T12:23:08.257248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Check image augmentation <a class=\"anchor\" id=\"25\"></a>","metadata":{}},{"cell_type":"code","source":"tmp_dl = DataLoader(tr_ds, batch_size=2)\nfig = plt.figure(figsize=(15, 6))\nnum_epochs = 5\nfor j in range(num_epochs):\n    img_batch, label_batch = next(iter(tmp_dl))\n    img = img_batch[0]\n    ax = fig.add_subplot(2, 5, j + 1)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_title(f'Epoch {j}:', size=15)\n    ax.imshow(img.permute(1, 2, 0))\n    \n    img = img_batch[1]\n    ax = fig.add_subplot(2, 5, j + 6)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img.permute(1, 2, 0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:23:08.259591Z","iopub.execute_input":"2023-10-12T12:23:08.260101Z","iopub.status.idle":"2023-10-12T12:23:09.624182Z","shell.execute_reply.started":"2023-10-12T12:23:08.260072Z","shell.execute_reply":"2023-10-12T12:23:09.623193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Build a ResNet-101 model from scratch<a class=\"anchor\" id=\"3\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Define model <a class=\"anchor\" id=\"31\"></a>","metadata":{}},{"cell_type":"code","source":"class Bottleneck(nn.Module):\n    # number of feature maps increases 4 fold after each Bottleneck block\n    expansion = 4\n\n    def __init__(self, inplanes, planes, stride=1, downsample=None):\n        super().__init__()\n        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(planes)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,\n                               padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(planes)\n        self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)\n        self.bn3 = nn.BatchNorm2d(planes * 4)\n        self.relu = nn.ReLU(inplace=True)\n        self.downsample = downsample\n        self.stride = stride\n\n    def forward(self, x):\n        residual = x\n\n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n\n        out = self.conv2(out)\n        out = self.bn2(out)\n        out = self.relu(out)\n\n        out = self.conv3(out)\n        out = self.bn3(out)\n\n        if self.downsample is not None:\n            residual = self.downsample(x)\n\n        out += residual\n        out = self.relu(out)\n\n        return out\n    \n\nclass ResNet(nn.Module):\n\n    def __init__(self, block, layers, num_classes, grayscale):\n        # this is the number of feature maps of the input\n        self.inplanes = 64\n        if grayscale:\n            in_dim = 1\n        else:\n            in_dim = 3\n        super().__init__()\n        \n        self.conv1 = nn.Conv2d(in_dim, self.inplanes, kernel_size=7, \n                               stride=2, padding=3, bias=False) \n        self.bn1 = nn.BatchNorm2d(self.inplanes)\n        self.relu = nn.ReLU(inplace=True)\n        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        self.layer1 = self._make_layer(block, 64, layers[0])\n        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)\n        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)\n        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)\n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        self.fc = nn.Linear(512 * block.expansion, num_classes)\n                \n        for m in self.modules():\n            if isinstance(m, nn.Conv2d):\n                nn.init.kaiming_normal_(m.weight, mode=\"fan_out\", nonlinearity=\"relu\")\n            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):\n                nn.init.constant_(m.weight, 1)\n                nn.init.constant_(m.bias, 0)\n\n    def _make_layer(self, block, planes, blocks, stride=1):\n        downsample = None\n        if stride != 1 or self.inplanes != planes * block.expansion:\n            downsample = nn.Sequential(\n                nn.Conv2d(self.inplanes, planes * block.expansion,\n                          kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(planes * block.expansion),\n            )\n\n        layers = []\n        layers.append(block(self.inplanes, planes, stride, downsample))\n        \n        # update the number of feature maps\n        self.inplanes = planes * block.expansion\n        for i in range(1, blocks):\n            layers.append(block(self.inplanes, planes))\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        x = self.conv1(x)\n        x = self.bn1(x)\n        x = self.relu(x)\n        x = self.maxpool(x)\n\n        x = self.layer1(x)\n        x = self.layer2(x)\n        x = self.layer3(x)\n        x = self.layer4(x)\n\n        x = self.avgpool(x)\n        x = x.view(x.size(0), -1)\n        logits = self.fc(x)\n#         probas = F.softmax(logits, dim=1)\n        return logits\n\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:27:44.803082Z","iopub.execute_input":"2023-10-12T12:27:44.803802Z","iopub.status.idle":"2023-10-12T12:27:44.819380Z","shell.execute_reply.started":"2023-10-12T12:27:44.803748Z","shell.execute_reply":"2023-10-12T12:27:44.818301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resnet101(num_classes, grayscale):\n    \"\"\"Constructs a ResNet-101 model.\"\"\"\n    model = ResNet(block=Bottleneck, \n                   layers=[3, 4, 23, 3],\n                   num_classes=NUM_CLASSES,\n                   grayscale=grayscale)\n    return model\n\nGRAYSCALE = False\nmodel = resnet101(num_classes = NUM_CLASSES, grayscale = GRAYSCALE)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:27:45.126478Z","iopub.execute_input":"2023-10-12T12:27:45.127285Z","iopub.status.idle":"2023-10-12T12:27:49.731845Z","shell.execute_reply.started":"2023-10-12T12:27:45.127256Z","shell.execute_reply":"2023-10-12T12:27:49.730970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Choose loss function, optimizer <a class=\"anchor\" id=\"33\"></a>","metadata":{}},{"cell_type":"code","source":"loss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:28:04.713736Z","iopub.execute_input":"2023-10-12T12:28:04.714102Z","iopub.status.idle":"2023-10-12T12:28:04.719874Z","shell.execute_reply.started":"2023-10-12T12:28:04.714075Z","shell.execute_reply":"2023-10-12T12:28:04.718954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Train model <a class=\"anchor\" id=\"34\"></a>","metadata":{}},{"cell_type":"code","source":"def train(model, num_epochs, train_dl, valid_dl):\n    model.to(device)\n    loss_hist_train = [0] * num_epochs\n    accuracy_hist_train = [0] * num_epochs\n    loss_hist_valid = [0] * num_epochs\n    accuracy_hist_valid = [0] * num_epochs\n    start_time = time.time() \n    for epoch in range(num_epochs):\n        \n        model = model.train()\n        for batch_idx, (x_batch, y_batch) in enumerate(train_dl):\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n            # forward\n            logits = model(x_batch)\n            loss = loss_fn(logits, y_batch)\n            _, pred = torch.max(logits, 1)\n            # backward\n            optimizer.zero_grad()\n            loss.backward()\n            # update\n            optimizer.step()\n            # logging\n            loss_hist_train[epoch] += loss.item() * y_batch.size(0)\n            is_correct = ((pred >= 0.5).float() == y_batch).float()\n            accuracy_hist_train[epoch] += is_correct.sum()\n            \n            if not batch_idx % 50:\n                print ('Epoch: %03d/%03d | Batch %03d/%03d | Loss: %.4f' \n                   %(epoch+1, num_epochs, batch_idx, \n                     len(train_dl), loss))\n            \n        loss_hist_train[epoch] /= len(train_dl.dataset)\n        accuracy_hist_train[epoch] /= len(train_dl.dataset)\n        \n        model = model.eval()\n        with torch.no_grad():\n            for x_batch, y_batch in valid_dl:\n                x_batch = x_batch.to(device)\n                y_batch = y_batch.to(device)\n                logits = model(x_batch)\n                loss = loss_fn(logits, y_batch)\n                _, pred = torch.max(logits, 1)\n                loss_hist_valid[epoch] += loss.item() * y_batch.size(0)\n                is_correct = ((pred >= 0.5).float() == y_batch).float()\n                accuracy_hist_valid[epoch] += is_correct.sum()\n            loss_hist_valid[epoch] /= len(valid_dl.dataset)\n            accuracy_hist_valid[epoch] /= len(valid_dl.dataset)\n        print(f'Epoch {epoch+1} accuracy: '\n              f'{accuracy_hist_train[epoch]:.4f} val_accuracy: '\n              f'{accuracy_hist_valid[epoch]:.4f}')\n        print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n    print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n    return loss_hist_train, loss_hist_valid, accuracy_hist_train, accuracy_hist_valid","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:28:06.501928Z","iopub.execute_input":"2023-10-12T12:28:06.502913Z","iopub.status.idle":"2023-10-12T12:28:06.514065Z","shell.execute_reply.started":"2023-10-12T12:28:06.502861Z","shell.execute_reply":"2023-10-12T12:28:06.512901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = train(model, NUM_EPOCHS, tr_dl, va_dl)\n\n# write history to file\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_from_scratch.csv')","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:28:08.853112Z","iopub.execute_input":"2023-10-12T12:28:08.853741Z","iopub.status.idle":"2023-10-12T12:28:42.274668Z","shell.execute_reply.started":"2023-10-12T12:28:08.853710Z","shell.execute_reply":"2023-10-12T12:28:42.273387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Visualize the learning curve  <a class=\"anchor\" id=\"35\"></a>","metadata":{}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_from_scratch.png', dpi = my_dpi);","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:02.34382Z","iopub.execute_input":"2023-09-15T19:46:02.344947Z","iopub.status.idle":"2023-09-15T19:46:02.838326Z","shell.execute_reply.started":"2023-09-15T19:46:02.344902Z","shell.execute_reply":"2023-09-15T19:46:02.836291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.6 Make predictions on test set  <a class=\"anchor\" id=\"36\"></a>","metadata":{}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Build a model using ResNet-152 pretrained weights  <a class=\"anchor\" id=\"4\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 4.1 Load pre-trained ResNet-152 <a class=\"anchor\" id=\"41\"></a>","metadata":{}},{"cell_type":"code","source":"resnet152 = models.resnet152(weights = True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-10-12T12:29:28.591079Z","iopub.execute_input":"2023-10-12T12:29:28.591773Z","iopub.status.idle":"2023-10-12T12:29:31.830723Z","shell.execute_reply.started":"2023-10-12T12:29:28.591744Z","shell.execute_reply":"2023-10-12T12:29:31.829723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Freeze early layers  <a class=\"anchor\" id=\"42\"></a>","metadata":{}},{"cell_type":"code","source":"for layer in resnet152.modules():\n    if not isinstance(layer, nn.Linear):\n        for param in layer.parameters():\n            param.requires_grad = False\n    else:\n        print(layer)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:29:57.735383Z","iopub.execute_input":"2023-10-12T12:29:57.735702Z","iopub.status.idle":"2023-10-12T12:29:57.747033Z","shell.execute_reply.started":"2023-10-12T12:29:57.735676Z","shell.execute_reply":"2023-10-12T12:29:57.745964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 Modify output layer <a class=\"anchor\" id=\"43\"></a>","metadata":{}},{"cell_type":"code","source":"resnet152.fc = nn.Linear(resnet152.fc.in_features, NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:30:12.575677Z","iopub.execute_input":"2023-10-12T12:30:12.576674Z","iopub.status.idle":"2023-10-12T12:30:12.582194Z","shell.execute_reply.started":"2023-10-12T12:30:12.576634Z","shell.execute_reply":"2023-10-12T12:30:12.580976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.4 Train model <a class=\"anchor\" id=\"44\"></a>","metadata":{}},{"cell_type":"code","source":"model = resnet152\nmodel.to(device)\n\nloss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n\nhistory = train(model, NUM_EPOCHS//2, tr_dl, va_dl)\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_transfer_learning.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-10-12T12:30:22.880490Z","iopub.execute_input":"2023-10-12T12:30:22.880958Z","iopub.status.idle":"2023-10-12T12:30:33.675503Z","shell.execute_reply.started":"2023-10-12T12:30:22.880915Z","shell.execute_reply":"2023-10-12T12:30:33.674164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.5 Visualize the learning curve <a class=\"anchor\" id=\"45\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:45.355441Z","iopub.execute_input":"2023-09-15T19:46:45.355826Z","iopub.status.idle":"2023-09-15T19:46:45.36278Z","shell.execute_reply.started":"2023-09-15T19:46:45.355793Z","shell.execute_reply":"2023-09-15T19:46:45.361834Z"}}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_transfer_learning.png', dpi = my_dpi);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.6 Make prediction on test set <a class=\"anchor\" id=\"46\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:48.989266Z","iopub.status.idle":"2023-09-15T19:50:48.9901Z","shell.execute_reply.started":"2023-09-15T19:50:48.989835Z","shell.execute_reply":"2023-09-15T19:50:48.989861Z"}}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}