{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-30T16:06:13.360727Z","iopub.execute_input":"2023-09-30T16:06:13.361544Z","iopub.status.idle":"2023-09-30T16:06:13.372540Z","shell.execute_reply.started":"2023-09-30T16:06:13.361509Z","shell.execute_reply":"2023-09-30T16:06:13.371723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Note\n\n* reference:\n    * from scratch\n        - https://github.com/pytorch/vision/blob/main/torchvision/models/googlenet.py\n    * transfer learning :\n        - https://cs231n.github.io/transfer-learning/\n        - https://pytorch.org/hub/pytorch_vision_googlenet/\n\n## Video walkthrough:\n- [link](https://youtube.com/shorts/L4xpzRLcCAU?feature=share)\n\n## Contents:\n* [Import packages](#1)\n    * [Seed everything](#11)\n    * [Set deterministic](#12)\n* [Preprocess image](#2)\n    * [Unzip](#21)    \n    * [Show image examples](#22)\n    * [Check image sizes](#23)    \n    * [Augmentation while loading](#24)\n    * [Check augmented images](#25)\n* [Build a GoogLeNet model from scratch](#3)\n    * [Define model](#31)\n    * [Choose loss function, optimizer](#33)\n    * [Train model](#34)\n    * [Visualize the learning curve](#35)\n    * [Make Predictions on test set](#36)\n* [Build a model using GoogLeNet pretrained weights](#4)\n    * [Load pre-trained GoogLeNet](#41)\n    * [Freeze early layers](#42)\n    * [Modify output layer](#43)\n    * [Train model](#44)\n    * [Visualize the learning curve](#45)\n    * [Make Predictions on test set](#46)","metadata":{}},{"cell_type":"markdown","source":"## 1. Import packages <a class=\"anchor\" id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nimport time\nimport multiprocessing\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nimport torch.nn.functional as F\nimport torchvision.models as models\n\n# from torchviz import make_dot\nimport random\n\nfrom dataclasses import make_dataclass\n\nfrom typing import Any, Callable, List, Optional, Tuple\nfrom torch import Tensor","metadata":{"execution":{"iopub.status.busy":"2023-09-30T16:21:40.348692Z","iopub.execute_input":"2023-09-30T16:21:40.349034Z","iopub.status.idle":"2023-09-30T16:21:40.354400Z","shell.execute_reply.started":"2023-09-30T16:21:40.348986Z","shell.execute_reply":"2023-09-30T16:21:40.353438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.1 Seed everything  <a class=\"anchor\" id=\"11\"></a>","metadata":{}},{"cell_type":"code","source":"SEED = 1\nos.environ[\"PL_GLOBAL_SEED\"] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:49.107384Z","iopub.execute_input":"2023-09-15T19:42:49.107758Z","iopub.status.idle":"2023-09-15T19:42:49.114839Z","shell.execute_reply.started":"2023-09-15T19:42:49.107728Z","shell.execute_reply":"2023-09-15T19:42:49.11386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Set cuDNN and PyTorch to be deterministic <a class=\"anchor\" id=\"12\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:05:19.156546Z","iopub.execute_input":"2023-09-15T11:05:19.156906Z","iopub.status.idle":"2023-09-15T11:05:19.161724Z","shell.execute_reply.started":"2023-09-15T11:05:19.156877Z","shell.execute_reply":"2023-09-15T11:05:19.16052Z"}}},{"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n# torch.set_deterministic(True)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:49.956441Z","iopub.execute_input":"2023-09-15T19:42:49.95758Z","iopub.status.idle":"2023-09-15T19:42:49.96272Z","shell.execute_reply.started":"2023-09-15T19:42:49.957542Z","shell.execute_reply":"2023-09-15T19:42:49.961643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Settings","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nLEARNING_RATE = 0.0001\nBATCH_SIZE = 256\nNUM_EPOCHS = 60\n\nNUM_CLASSES = 2\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:50.697514Z","iopub.execute_input":"2023-09-15T19:42:50.697883Z","iopub.status.idle":"2023-09-15T19:42:50.702874Z","shell.execute_reply.started":"2023-09-15T19:42:50.697852Z","shell.execute_reply":"2023-09-15T19:42:50.701978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Preprocess image <a class=\"anchor\" id=\"2\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Unzip <a class=\"anchor\" id=\"21\"></a>","metadata":{}},{"cell_type":"code","source":"base_dir = '../input/dogs-vs-cats-redux-kernels-edition'\ntrain_dir = '../data/train'\ntest_dir = '../data/test'\n\nwith zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:\n    train_zip.extractall('../data')\n    \nwith zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:\n    test_zip.extractall('../data')\n    \ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\nlen(train_list), len(test_list)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:51.636262Z","iopub.execute_input":"2023-09-15T19:42:51.636924Z","iopub.status.idle":"2023-09-15T19:43:08.575713Z","shell.execute_reply.started":"2023-09-15T19:42:51.636891Z","shell.execute_reply":"2023-09-15T19:43:08.574836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Show training image examples <a class=\"anchor\" id=\"22\"></a>","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure(figsize = (15,6))\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = Image.open(train_list[idx])\n    img_path = train_list[idx]\n    label = img_path.split('/')[-1].split('.')[0]\n    plt.imshow(img)\n    plt.title(label, size=15)\n    i+=1\n    \nplt.tight_layout()\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:08.577887Z","iopub.execute_input":"2023-09-15T19:43:08.578265Z","iopub.status.idle":"2023-09-15T19:43:11.524123Z","shell.execute_reply.started":"2023-09-15T19:43:08.578231Z","shell.execute_reply":"2023-09-15T19:43:11.523032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Check the original sizes of the training images and channels <a class=\"anchor\" id=\"23\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:14:14.711891Z","iopub.execute_input":"2023-07-02T22:14:14.712268Z","iopub.status.idle":"2023-07-02T22:14:14.717015Z","shell.execute_reply.started":"2023-07-02T22:14:14.712238Z","shell.execute_reply":"2023-07-02T22:14:14.715667Z"}}},{"cell_type":"code","source":"h_list = []\nw_list = []\nband_set = set()\nfor train_img in train_list:\n    img = Image.open(train_img)\n    w, h = img.size\n    h_list.append(h)\n    w_list.append(w)\n    band_set.add(img.getbands())\n    \nw_med, h_med = np.percentile(w_list, q = 80), np.percentile(h_list, q = 80)\nprint('80 percentile width of all the training images = ', w_med, \n      '\\n80 percentile height of all the training images =', h_med)\nprint('channels of the training images:', band_set)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:11.525808Z","iopub.execute_input":"2023-09-15T19:43:11.526751Z","iopub.status.idle":"2023-09-15T19:43:14.515415Z","shell.execute_reply.started":"2023-09-15T19:43:11.526711Z","shell.execute_reply":"2023-09-15T19:43:14.514177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Image augmentation during data loading <a class=\"anchor\" id=\"24\"></a>","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.manual_seed(SEED)\nif device =='cuda':\n    torch.cuda.manual_seed_all(SEED)\nprint(\"device:\", device)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.518767Z","iopub.execute_input":"2023-09-15T19:43:14.519459Z","iopub.status.idle":"2023-09-15T19:43:14.531995Z","shell.execute_reply.started":"2023-09-15T19:43:14.519419Z","shell.execute_reply":"2023-09-15T19:43:14.530926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    'train': transforms.Compose([\n        transforms.RandomResizedCrop([512,512]),\n        transforms.RandomHorizontalFlip(),\n        transforms.Resize([256,256]),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize([256,256]),\n        transforms.ToTensor()\n    ])\n}","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.533565Z","iopub.execute_input":"2023-09-15T19:43:14.53415Z","iopub.status.idle":"2023-09-15T19:43:14.580162Z","shell.execute_reply.started":"2023-09-15T19:43:14.534109Z","shell.execute_reply":"2023-09-15T19:43:14.579031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        if self.transform is not None:\n            img = self.transform(img)\n        label = img_path.split('/')[-1].split('.')[0]\n        if label == 'dog':\n            label = 1\n        elif label == 'cat':\n            label = 0\n            \n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.582941Z","iopub.execute_input":"2023-09-15T19:43:14.583719Z","iopub.status.idle":"2023-09-15T19:43:14.593698Z","shell.execute_reply.started":"2023-09-15T19:43:14.583681Z","shell.execute_reply":"2023-09-15T19:43:14.592719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trlist, valist = train_test_split(train_list, test_size=0.1)\n\ntr_ds = dataset(trlist, transform=data_transforms['train'])\nva_ds = dataset(valist, transform=data_transforms['test'])\nte_ds = dataset(test_list, transform=data_transforms['test'])\n\n# tr_ds = Subset(tr_ds, torch.arange(1000))\n# va_ds = Subset(va_ds, torch.arange(100))\n\nnum_workers = multiprocessing.cpu_count()\ntr_dl = DataLoader(tr_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers = num_workers, pin_memory = True)\nva_dl = DataLoader(va_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.595519Z","iopub.execute_input":"2023-09-15T19:43:14.595913Z","iopub.status.idle":"2023-09-15T19:43:14.620873Z","shell.execute_reply.started":"2023-09-15T19:43:14.595878Z","shell.execute_reply":"2023-09-15T19:43:14.619807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Check image augmentation <a class=\"anchor\" id=\"25\"></a>","metadata":{}},{"cell_type":"code","source":"tmp_dl = DataLoader(tr_ds, batch_size=2)\nfig = plt.figure(figsize=(15, 6))\nnum_epochs = 5\nfor j in range(num_epochs):\n    img_batch, label_batch = next(iter(tmp_dl))\n    img = img_batch[0]\n    ax = fig.add_subplot(2, 5, j + 1)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_title(f'Epoch {j}:', size=15)\n    ax.imshow(img.permute(1, 2, 0))\n    \n    img = img_batch[1]\n    ax = fig.add_subplot(2, 5, j + 6)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img.permute(1, 2, 0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.623134Z","iopub.execute_input":"2023-09-15T19:43:14.623531Z","iopub.status.idle":"2023-09-15T19:43:15.606003Z","shell.execute_reply.started":"2023-09-15T19:43:14.623492Z","shell.execute_reply":"2023-09-15T19:43:15.605155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Build a GoogLeNet model from scratch<a class=\"anchor\" id=\"3\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Define model <a class=\"anchor\" id=\"31\"></a>","metadata":{}},{"cell_type":"code","source":"Row = make_dataclass(\"Row\",\n        [\"Layer\", \"Type\", \"Out_Channels\",(\"Depth\",int)])\ndf = pd.DataFrame([Row(\"In\",\"Input\",3,0),\n        Row(\"C1\",\"Convolution\",64, 1),\n        Row(\"P2\",\"Max Pooling\",64, 0),\n        Row(\"C3\",\"Convolution\",192, 2),\n        Row(\"P4\",\"Max Pooling\",192, 0),\n        Row(\"I5\",\"Inception\", 256, 2),\n        Row(\"I6\",\"Inception\", 480, 2),\n        Row(\"P7\",\"Max Pooling\",480, 0),\n        Row(\"I8\",\"Inception\", 512, 2),\n        Row(\"I9\",\"Inception\", 512, 2),\n        Row(\"I10\",\"Inception\", 512, 2),\n        Row(\"I11\",\"Inception\", 528, 2),\n        Row(\"I12\",\"Inception\", 832, 2),\n        Row(\"P13\",\"Max Pooling\",832, 0),\n        Row(\"I14\",\"Inception\", 832, 2),\n        Row(\"I15\",\"Inception\", 1024, 2),\n        Row(\"P16\",\"Avg Pooling\",1024, 0),\n        Row(\"D17\",\"Dropout\", 1024, 0),\n        Row(\"F18\",\"Fully Connected\",'Num Classes',1),\n        ])\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2023-09-30T16:21:12.627479Z","iopub.execute_input":"2023-09-30T16:21:12.627798Z","iopub.status.idle":"2023-09-30T16:21:12.645146Z","shell.execute_reply.started":"2023-09-30T16:21:12.627772Z","shell.execute_reply":"2023-09-30T16:21:12.644096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Inception(nn.Module):\n    def __init__(self, in_channels: int, ch1x1:int, ch3x3red:int, ch3x3:int,\n                ch5x5red:int, ch5x5:int, pool_proj:int) -> None:\n        super().__init__()\n        self.branch1 = nn.Conv2d(in_channels, ch1x1, kernel_size=1)\n        \n        self.branch2 = nn.Sequential(\n            nn.Conv2d(in_channels,ch3x3red, kernel_size=1),\n            nn.Conv2d(ch3x3red, ch3x3, kernel_size=3, padding=1)\n        )\n        \n        self.branch3 = nn.Sequential(\n            nn.Conv2d(in_channels, ch5x5red, kernel_size=1),\n            nn.Conv2d(ch5x5red, ch5x5, kernel_size=5, padding=2)\n        )\n        \n        self.branch4 = nn.Sequential(\n            nn.MaxPool2d(kernel_size=3, stride=1, padding=1, ceil_mode = True),\n            nn.Conv2d(in_channels, pool_proj, kernel_size = 1)\n        )\n        \n    def _forward(self, x: Tensor) -> List[Tensor]:\n        branch1 = self.branch1(x)\n        branch2 = self.branch2(x)\n        branch3 = self.branch3(x)\n        branch4 = self.branch4(x)\n        \n        outputs = [branch1, branch2, branch3, branch4]\n        return outputs\n    \n    def forward(self, x: Tensor) -> Tensor:\n        outputs = self._forward(x)\n        return torch.cat(outputs, 1) # concatenate along the channels dimension\n\nclass GoogLeNet(nn.Module):\n    def __init__(self, num_classes:int = 2, dropout_proba: float = 0.4) -> None:\n        super().__init__()\n        self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3)\n        self.maxpool1 = nn.MaxPool2d(3, stride=2, ceil_mode=True)\n        self.conv2 = nn.Conv2d(64, 64, kernel_size=1)\n        self.conv3 = nn.Conv2d(64, 192, kernel_size=3, padding=1)\n        self.maxpool2 = nn.MaxPool2d(3, stride=2, ceil_mode=True)\n        # inception 3a out_channels = 64+128+32+32 = 256\n        self.inception3a = Inception(192, 64, 96, 128, 16, 32, 32)   \n        # inception 3b out_channels = 128+192+96+64 = 480\n        self.inception3b = Inception(256, 128, 128, 192, 32, 96, 64)\n        self.maxpool3 = nn.MaxPool2d(3, stride=2, ceil_mode=True)\n        \n        # inception 4a out_channels = 192+208+48+64 = 512\n        self.inception4a = Inception(480, 192, 96, 208, 16, 48, 64)\n        # inception 4b out_channels = 160+224+64+64 = 512\n        self.inception4b = Inception(512, 160, 112, 224, 24, 64, 64)\n        # inception 4c out_channels = 128+256+64+64 = 512\n        self.inception4c = Inception(512, 128, 128, 256, 24, 64, 64)\n        # inception 4d out_channels = 112+288+64+64 = 528\n        self.inception4d = Inception(512, 112, 144, 288, 32, 64, 64)\n        # inception 4e out_channels = 256+320+128+128 = 832\n        self.inception4e = Inception(528, 256, 160, 320, 32, 128, 128)\n        self.maxpool4 = nn.MaxPool2d(2, stride=2, ceil_mode=True)\n        \n        # inception 5a out_channels = 256+320+128+128 = 832\n        self.inception5a = Inception(832, 256, 160, 320, 32, 128, 128)\n        # inception 5b out_channels = 384+384+128+128 = 1024\n        self.inception5b = Inception(832, 384, 192, 384, 48, 128, 128)\n        \n        self.avgpool = nn.AdaptiveAvgPool2d((1,1))\n        self.dropout = nn.Dropout(p = dropout_proba)\n        self.fc = nn.Linear(1024, num_classes)\n        \n    def forward(self, x: Tensor) -> Tensor:\n        # N x 3 x 224 x 224\n        x = self.conv1(x)\n        # N x 64 x 112 x 112\n        x = self.maxpool1(x)\n        # N x 64 x 56 x 56\n        x = self.conv2(x)\n        # N x 64 x 56 x 56\n        x = self.conv3(x)\n        # N x 192 x 56 x 56\n        x = self.maxpool2(x)\n\n        # N x 192 x 28 x 28\n        x = self.inception3a(x)\n        # N x 256 x 28 x 28\n        x = self.inception3b(x)\n        # N x 480 x 28 x 28\n        x = self.maxpool3(x)\n        # N x 480 x 14 x 14\n        x = self.inception4a(x)\n        # N x 512 x 14 x 14\n        x = self.inception4b(x)\n        # N x 512 x 14 x 14\n        x = self.inception4c(x)\n        # N x 512 x 14 x 14\n        x = self.inception4d(x)\n        # N x 528 x 14 x 14\n        x = self.inception4e(x)\n        # N x 832 x 14 x 14\n        x = self.maxpool4(x)\n        # N x 832 x 7 x 7\n        x = self.inception5a(x)\n        # N x 832 x 7 x 7\n        x = self.inception5b(x)\n        # N x 1024 x 7 x 7\n\n        x = self.avgpool(x)\n        # N x 1024 x 1 x 1\n        x = torch.flatten(x, 1)\n        # N x 1024\n        x = self.dropout(x)\n        x = self.fc(x)\n        # N x 1000 (num_classes)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-09-30T16:21:46.728529Z","iopub.execute_input":"2023-09-30T16:21:46.728871Z","iopub.status.idle":"2023-09-30T16:21:46.742976Z","shell.execute_reply.started":"2023-09-30T16:21:46.728843Z","shell.execute_reply":"2023-09-30T16:21:46.742152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = GoogLeNet(num_classes = NUM_CLASSES)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-09-30T16:21:50.077753Z","iopub.execute_input":"2023-09-30T16:21:50.078099Z","iopub.status.idle":"2023-09-30T16:21:50.104233Z","shell.execute_reply.started":"2023-09-30T16:21:50.078073Z","shell.execute_reply":"2023-09-30T16:21:50.102861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Choose loss function, optimizer <a class=\"anchor\" id=\"33\"></a>","metadata":{}},{"cell_type":"code","source":"loss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:49:28.223595Z","iopub.execute_input":"2023-09-15T11:49:28.223977Z","iopub.status.idle":"2023-09-15T11:49:28.22903Z","shell.execute_reply.started":"2023-09-15T11:49:28.223946Z","shell.execute_reply":"2023-09-15T11:49:28.228036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Train model <a class=\"anchor\" id=\"34\"></a>","metadata":{}},{"cell_type":"code","source":"def train(model, num_epochs, train_dl, valid_dl):\n    model.to(device)\n    loss_hist_train = [0] * num_epochs\n    accuracy_hist_train = [0] * num_epochs\n    loss_hist_valid = [0] * num_epochs\n    accuracy_hist_valid = [0] * num_epochs\n    start_time = time.time() \n    for epoch in range(num_epochs):\n        \n        model = model.train()\n        for batch_idx, (x_batch, y_batch) in enumerate(train_dl):\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n            # forward\n            logits = model(x_batch)\n            loss = loss_fn(logits, y_batch)\n            _, pred = torch.max(logits, 1)\n            # backward\n            optimizer.zero_grad()\n            loss.backward()\n            # update\n            optimizer.step()\n            # logging\n            loss_hist_train[epoch] += loss.item() * y_batch.size(0)\n            is_correct = ((pred >= 0.5).float() == y_batch).float()\n            accuracy_hist_train[epoch] += is_correct.sum()\n            \n            if not batch_idx % 50:\n                print ('Epoch: %03d/%03d | Batch %03d/%03d | Loss: %.4f' \n                   %(epoch+1, num_epochs, batch_idx, \n                     len(train_dl), loss))\n            \n        loss_hist_train[epoch] /= len(train_dl.dataset)\n        accuracy_hist_train[epoch] /= len(train_dl.dataset)\n        \n        model = model.eval()\n        with torch.no_grad():\n            for x_batch, y_batch in valid_dl:\n                x_batch = x_batch.to(device)\n                y_batch = y_batch.to(device)\n                logits = model(x_batch)\n                loss = loss_fn(logits, y_batch)\n                _, pred = torch.max(logits, 1)\n                loss_hist_valid[epoch] += loss.item() * y_batch.size(0)\n                is_correct = ((pred >= 0.5).float() == y_batch).float()\n                accuracy_hist_valid[epoch] += is_correct.sum()\n            loss_hist_valid[epoch] /= len(valid_dl.dataset)\n            accuracy_hist_valid[epoch] /= len(valid_dl.dataset)\n        print(f'Epoch {epoch+1} accuracy: '\n              f'{accuracy_hist_train[epoch]:.4f} val_accuracy: '\n              f'{accuracy_hist_valid[epoch]:.4f}')\n        print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n    print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n    return loss_hist_train, loss_hist_valid, accuracy_hist_train, accuracy_hist_valid","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:18.138556Z","iopub.execute_input":"2023-09-15T19:50:18.138983Z","iopub.status.idle":"2023-09-15T19:50:18.15693Z","shell.execute_reply.started":"2023-09-15T19:50:18.138927Z","shell.execute_reply":"2023-09-15T19:50:18.155945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = train(model, NUM_EPOCHS, tr_dl, va_dl)\n\n# write history to file\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_from_scratch.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:55:33.640942Z","iopub.execute_input":"2023-09-15T11:55:33.641313Z","iopub.status.idle":"2023-09-15T11:56:02.833703Z","shell.execute_reply.started":"2023-09-15T11:55:33.641275Z","shell.execute_reply":"2023-09-15T11:56:02.832174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Visualize the learning curve  <a class=\"anchor\" id=\"35\"></a>","metadata":{}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_from_scratch.png', dpi = my_dpi);","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:02.34382Z","iopub.execute_input":"2023-09-15T19:46:02.344947Z","iopub.status.idle":"2023-09-15T19:46:02.838326Z","shell.execute_reply.started":"2023-09-15T19:46:02.344902Z","shell.execute_reply":"2023-09-15T19:46:02.836291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.6 Make predictions on test set  <a class=\"anchor\" id=\"36\"></a>","metadata":{}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Build a model using GoogLeNet pretrained weights  <a class=\"anchor\" id=\"4\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 4.1 Load pre-trained GoogLeNet <a class=\"anchor\" id=\"41\"></a>","metadata":{}},{"cell_type":"code","source":"googlenet = models.googlenet(weights = True)\ngooglenet","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-30T16:21:57.835449Z","iopub.execute_input":"2023-09-30T16:21:57.835792Z","iopub.status.idle":"2023-09-30T16:21:58.489924Z","shell.execute_reply.started":"2023-09-30T16:21:57.835763Z","shell.execute_reply":"2023-09-30T16:21:58.489070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Freeze early layers  <a class=\"anchor\" id=\"42\"></a>","metadata":{}},{"cell_type":"code","source":"for layer in googlenet.modules():\n    if not isinstance(layer, nn.Linear):\n        for param in layer.parameters():\n            param.requires_grad = False\n    else:\n        print(layer)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:30.452995Z","iopub.execute_input":"2023-09-15T19:43:30.453373Z","iopub.status.idle":"2023-09-15T19:43:30.459512Z","shell.execute_reply.started":"2023-09-15T19:43:30.453342Z","shell.execute_reply":"2023-09-15T19:43:30.458085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 Modify output layer <a class=\"anchor\" id=\"43\"></a>","metadata":{}},{"cell_type":"code","source":"googlenet.fc = nn.Linear(googlenet.fc.in_features, NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:55:29.990831Z","iopub.execute_input":"2023-09-15T19:55:29.991424Z","iopub.status.idle":"2023-09-15T19:55:29.997933Z","shell.execute_reply.started":"2023-09-15T19:55:29.991388Z","shell.execute_reply":"2023-09-15T19:55:29.996871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.4 Train model <a class=\"anchor\" id=\"44\"></a>","metadata":{}},{"cell_type":"code","source":"model = googlenet\nmodel.to(device)\n\nloss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n\nhistory = train(model, NUM_EPOCHS//2, tr_dl, va_dl)\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_transfer_learning.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:21.07112Z","iopub.execute_input":"2023-09-15T19:50:21.071491Z","iopub.status.idle":"2023-09-15T19:50:48.987589Z","shell.execute_reply.started":"2023-09-15T19:50:21.071462Z","shell.execute_reply":"2023-09-15T19:50:48.985868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.5 Visualize the learning curve <a class=\"anchor\" id=\"45\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:45.355441Z","iopub.execute_input":"2023-09-15T19:46:45.355826Z","iopub.status.idle":"2023-09-15T19:46:45.36278Z","shell.execute_reply.started":"2023-09-15T19:46:45.355793Z","shell.execute_reply":"2023-09-15T19:46:45.361834Z"}}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_transfer_learning.png', dpi = my_dpi);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.6 Make prediction on test set <a class=\"anchor\" id=\"46\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:48.989266Z","iopub.status.idle":"2023-09-15T19:50:48.9901Z","shell.execute_reply.started":"2023-09-15T19:50:48.989835Z","shell.execute_reply":"2023-09-15T19:50:48.989861Z"}}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}