{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-09-15T19:42:41.870784Z","iopub.execute_input":"2023-09-15T19:42:41.871184Z","iopub.status.idle":"2023-09-15T19:42:41.880794Z","shell.execute_reply.started":"2023-09-15T19:42:41.871154Z","shell.execute_reply":"2023-09-15T19:42:41.879630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Note\n\n* reference:\n    * from scratch\n        - https://d2l.ai/chapter_convolutional-modern/alexnet.html\n        - https://github.com/rasbt/deeplearning-models/blob/master/pytorch_ipynb/cnn/cnn-alexnet-cifar10.ipynb\n        - https://github.com/pytorch/vision/blob/main/torchvision/models/alexnet.py\n    * transfer learning :\n        - https://cs231n.github.io/transfer-learning/\n        - https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html\n\n## Video walkthrough:\n- [link](https://youtube.com/shorts/EyPobyr_0xU?feature=share)\n\n## Contents:\n* [Import packages](#1)\n    * [Seed everything](#11)\n    * [Set deterministic](#12)\n* [Preprocess image](#2)\n    * [Unzip](#21)    \n    * [Show image examples](#22)\n    * [Check image sizes](#23)    \n    * [Augmentation while loading](#24)\n    * [Check augmented images](#25)\n* [Build a AlexNet model](#3)\n    * [Define model](#31)\n    * [Choose loss function, optimizer](#33)\n    * [Train model](#34)\n    * [Visualize the learning curve](#35)\n    * [Make Predictions on test set](#36)\n* [Build a model using AlexNet weights](#4)\n    * [Load pre-trained AlexNet](#41)\n    * [Freeze early layers](#42)\n    * [Modify output layer](#43)\n    * [Train model](#44)\n    * [Visualize the learning curve](#45)\n    * [Make Predictions on test set](#46)","metadata":{}},{"cell_type":"markdown","source":"## 1. Import packages <a class=\"anchor\" id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport glob\nimport time\nimport multiprocessing\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nimport torch.nn.functional as F\nimport torchvision.models as models\n\n# from torchviz import make_dot\nimport random\n\nfrom dataclasses import make_dataclass","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:47.872256Z","iopub.execute_input":"2023-09-15T19:42:47.872686Z","iopub.status.idle":"2023-09-15T19:42:47.884047Z","shell.execute_reply.started":"2023-09-15T19:42:47.872649Z","shell.execute_reply":"2023-09-15T19:42:47.882863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.1 Seed everything  <a class=\"anchor\" id=\"11\"></a>","metadata":{}},{"cell_type":"code","source":"SEED = 1\nos.environ[\"PL_GLOBAL_SEED\"] = str(SEED)\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:49.107384Z","iopub.execute_input":"2023-09-15T19:42:49.107758Z","iopub.status.idle":"2023-09-15T19:42:49.114839Z","shell.execute_reply.started":"2023-09-15T19:42:49.107728Z","shell.execute_reply":"2023-09-15T19:42:49.113860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Set cuDNN and PyTorch to be deterministic <a class=\"anchor\" id=\"12\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:05:19.156546Z","iopub.execute_input":"2023-09-15T11:05:19.156906Z","iopub.status.idle":"2023-09-15T11:05:19.161724Z","shell.execute_reply.started":"2023-09-15T11:05:19.156877Z","shell.execute_reply":"2023-09-15T11:05:19.16052Z"}}},{"cell_type":"code","source":"if torch.cuda.is_available():\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\n# torch.set_deterministic(True)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:49.956441Z","iopub.execute_input":"2023-09-15T19:42:49.957580Z","iopub.status.idle":"2023-09-15T19:42:49.962720Z","shell.execute_reply.started":"2023-09-15T19:42:49.957542Z","shell.execute_reply":"2023-09-15T19:42:49.961643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.3 Settings","metadata":{}},{"cell_type":"code","source":"# Hyperparameters\nLEARNING_RATE = 0.0001\nBATCH_SIZE = 256\nNUM_EPOCHS = 60\n\nNUM_CLASSES = 2\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:50.697514Z","iopub.execute_input":"2023-09-15T19:42:50.697883Z","iopub.status.idle":"2023-09-15T19:42:50.702874Z","shell.execute_reply.started":"2023-09-15T19:42:50.697852Z","shell.execute_reply":"2023-09-15T19:42:50.701978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Preprocess image <a class=\"anchor\" id=\"2\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Unzip <a class=\"anchor\" id=\"21\"></a>","metadata":{}},{"cell_type":"code","source":"base_dir = '../input/dogs-vs-cats-redux-kernels-edition'\ntrain_dir = '../data/train'\ntest_dir = '../data/test'\n\nwith zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:\n    train_zip.extractall('../data')\n    \nwith zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:\n    test_zip.extractall('../data')\n    \ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\nlen(train_list), len(test_list)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:42:51.636262Z","iopub.execute_input":"2023-09-15T19:42:51.636924Z","iopub.status.idle":"2023-09-15T19:43:08.575713Z","shell.execute_reply.started":"2023-09-15T19:42:51.636891Z","shell.execute_reply":"2023-09-15T19:43:08.574836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Show training image examples <a class=\"anchor\" id=\"22\"></a>","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure(figsize = (15,6))\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = Image.open(train_list[idx])\n    img_path = train_list[idx]\n    label = img_path.split('/')[-1].split('.')[0]\n    plt.imshow(img)\n    plt.title(label, size=15)\n    i+=1\n    \nplt.tight_layout()\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:08.577887Z","iopub.execute_input":"2023-09-15T19:43:08.578265Z","iopub.status.idle":"2023-09-15T19:43:11.524123Z","shell.execute_reply.started":"2023-09-15T19:43:08.578231Z","shell.execute_reply":"2023-09-15T19:43:11.523032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Check the original sizes of the training images and channels <a class=\"anchor\" id=\"23\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:14:14.711891Z","iopub.execute_input":"2023-07-02T22:14:14.712268Z","iopub.status.idle":"2023-07-02T22:14:14.717015Z","shell.execute_reply.started":"2023-07-02T22:14:14.712238Z","shell.execute_reply":"2023-07-02T22:14:14.715667Z"}}},{"cell_type":"code","source":"h_list = []\nw_list = []\nband_set = set()\nfor train_img in train_list:\n    img = Image.open(train_img)\n    w, h = img.size\n    h_list.append(h)\n    w_list.append(w)\n    band_set.add(img.getbands())\n    \nw_med, h_med = np.percentile(w_list, q = 80), np.percentile(h_list, q = 80)\nprint('80 percentile width of all the training images = ', w_med, \n      '\\n80 percentile height of all the training images =', h_med)\nprint('channels of the training images:', band_set)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:11.525808Z","iopub.execute_input":"2023-09-15T19:43:11.526751Z","iopub.status.idle":"2023-09-15T19:43:14.515415Z","shell.execute_reply.started":"2023-09-15T19:43:11.526711Z","shell.execute_reply":"2023-09-15T19:43:14.514177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Image augmentation during data loading <a class=\"anchor\" id=\"24\"></a>","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.manual_seed(SEED)\nif device =='cuda':\n    torch.cuda.manual_seed_all(SEED)\nprint(\"device:\", device)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.518767Z","iopub.execute_input":"2023-09-15T19:43:14.519459Z","iopub.status.idle":"2023-09-15T19:43:14.531995Z","shell.execute_reply.started":"2023-09-15T19:43:14.519419Z","shell.execute_reply":"2023-09-15T19:43:14.530926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    'train': transforms.Compose([\n        transforms.RandomResizedCrop([512,512]),\n        transforms.RandomHorizontalFlip(),\n        transforms.Resize([256,256]),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize([256,256]),\n        transforms.ToTensor()\n    ])\n}","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.533565Z","iopub.execute_input":"2023-09-15T19:43:14.534150Z","iopub.status.idle":"2023-09-15T19:43:14.580162Z","shell.execute_reply.started":"2023-09-15T19:43:14.534109Z","shell.execute_reply":"2023-09-15T19:43:14.579031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        if self.transform is not None:\n            img = self.transform(img)\n        label = img_path.split('/')[-1].split('.')[0]\n        if label == 'dog':\n            label = 1\n        elif label == 'cat':\n            label = 0\n            \n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.582941Z","iopub.execute_input":"2023-09-15T19:43:14.583719Z","iopub.status.idle":"2023-09-15T19:43:14.593698Z","shell.execute_reply.started":"2023-09-15T19:43:14.583681Z","shell.execute_reply":"2023-09-15T19:43:14.592719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trlist, valist = train_test_split(train_list, test_size=0.1)\n\ntr_ds = dataset(trlist, transform=data_transforms['train'])\nva_ds = dataset(valist, transform=data_transforms['test'])\nte_ds = dataset(test_list, transform=data_transforms['test'])\n\n# tr_ds = Subset(tr_ds, torch.arange(1000))\n# va_ds = Subset(va_ds, torch.arange(100))\n\nnum_workers = multiprocessing.cpu_count()\ntr_dl = DataLoader(tr_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers = num_workers, pin_memory = True)\nva_dl = DataLoader(va_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.595519Z","iopub.execute_input":"2023-09-15T19:43:14.595913Z","iopub.status.idle":"2023-09-15T19:43:14.620873Z","shell.execute_reply.started":"2023-09-15T19:43:14.595878Z","shell.execute_reply":"2023-09-15T19:43:14.619807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Check image augmentation <a class=\"anchor\" id=\"25\"></a>","metadata":{}},{"cell_type":"code","source":"tmp_dl = DataLoader(tr_ds, batch_size=2)\nfig = plt.figure(figsize=(15, 6))\nnum_epochs = 5\nfor j in range(num_epochs):\n    img_batch, label_batch = next(iter(tmp_dl))\n    img = img_batch[0]\n    ax = fig.add_subplot(2, 5, j + 1)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_title(f'Epoch {j}:', size=15)\n    ax.imshow(img.permute(1, 2, 0))\n    \n    img = img_batch[1]\n    ax = fig.add_subplot(2, 5, j + 6)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img.permute(1, 2, 0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:14.623134Z","iopub.execute_input":"2023-09-15T19:43:14.623531Z","iopub.status.idle":"2023-09-15T19:43:15.606003Z","shell.execute_reply.started":"2023-09-15T19:43:14.623492Z","shell.execute_reply":"2023-09-15T19:43:15.605155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Build a AlexNet model <a class=\"anchor\" id=\"3\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Define model <a class=\"anchor\" id=\"31\"></a>","metadata":{}},{"cell_type":"code","source":"Row = make_dataclass(\"Row\",\n        [\"Layer\", \"Type\", (\"Out_Channels\",int),\n        \"Kernel_Size\",(\"Stride\",int),(\"Padding\",int),\"Activation\"])\ndf = pd.DataFrame([Row(\"In\",\"Input\",3,None,None,None,None),\n        Row(\"C1\",\"Convolution\",64,'11x11',4,2,'ReLU'),\n        Row(\"P2\",\"Max Pooling\",64,'3x3',2,0,None),\n        Row(\"C3\",\"Convolution\",192,'5x5',1,2,'ReLU'),\n        Row(\"P4\",\"Max Pooling\",192,'3x3',2,0,None),\n        Row(\"C5\",\"Convolution\",384,'3x3',1,1,'ReLU'),\n        Row(\"C6\",\"Convolution\",256,'3x3',1,1,'ReLU'),\n        Row(\"C7\",\"Convolution\",256,'3x3',1,1,'ReLU'),\n        Row(\"P8\",\"Max Pooling\",256,'3x3',2,0,None),\n        Row(\"P9\",\"Avg Pooling\",256,None,None,None,None),\n        Row(\"F10\",\"Flatten\",9216,None,None,None,None),\n        Row(\"F11\",\"Fully Connected\",4096,None,None,None,\"ReLU\"),\n        Row(\"F12\",\"Fully Connected\",4096,None,None,None,\"ReLU\"),\n        Row(\"Out\",\"Fully Connected\",'Num Classes',None,None,None,\"Softmax\"),\n        ])\ndisplay(df)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T19:04:02.765546Z","iopub.execute_input":"2023-09-16T19:04:02.766131Z","iopub.status.idle":"2023-09-16T19:04:03.273077Z","shell.execute_reply.started":"2023-09-16T19:04:02.766091Z","shell.execute_reply":"2023-09-16T19:04:03.271127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AlexNet(nn.Module):\n    \n    def __init__(self, num_classes):\n        super().__init__()\n        \n        # conv layers\n        self.features = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=11,\n                     stride=4, padding=2),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2), # padding = valid\n            \n            nn.Conv2d(in_channels=64, out_channels=192, kernel_size=5,\n                      padding=2),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2),\n            \n            nn.Conv2d(in_channels=192, out_channels=384, kernel_size=3,\n                     padding=1),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(in_channels=384, out_channels=256, kernel_size=3,\n                     padding=1),\n            nn.ReLU(inplace=True),\n            \n            nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3,\n                     padding=1),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2)\n        )\n        \n        # AdaptiveAvgPool2d() allows flexible input sizes\n        # the output size is needed for the linear layers next\n        self.avgpool = nn.AdaptiveAvgPool2d((6,6))\n        \n        # linear layers\n        self.classifier = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(in_features=256 * 6 * 6, out_features=4096),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.5),\n            nn.Linear(4096,4096),\n            nn.ReLU(inplace=True),\n            nn.Linear(4096, num_classes)\n        )\n    \n    def forward(self, x):\n        x = self.features(x)\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1) # flatten from the index=1 dim to the end\n        logits = self.classifier(x)\n        probas = F.softmax(logits, dim=1) # softmax computed per example\n        return logits","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:48:47.19352Z","iopub.execute_input":"2023-09-15T11:48:47.194315Z","iopub.status.idle":"2023-09-15T11:48:47.206962Z","shell.execute_reply.started":"2023-09-15T11:48:47.194271Z","shell.execute_reply":"2023-09-15T11:48:47.205992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AlexNet(num_classes = NUM_CLASSES)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:48:48.234822Z","iopub.execute_input":"2023-09-15T11:48:48.235196Z","iopub.status.idle":"2023-09-15T11:48:48.780109Z","shell.execute_reply.started":"2023-09-15T11:48:48.23516Z","shell.execute_reply":"2023-09-15T11:48:48.779003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Choose loss function, optimizer <a class=\"anchor\" id=\"33\"></a>","metadata":{}},{"cell_type":"code","source":"loss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:49:28.223595Z","iopub.execute_input":"2023-09-15T11:49:28.223977Z","iopub.status.idle":"2023-09-15T11:49:28.22903Z","shell.execute_reply.started":"2023-09-15T11:49:28.223946Z","shell.execute_reply":"2023-09-15T11:49:28.228036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Train model <a class=\"anchor\" id=\"34\"></a>","metadata":{}},{"cell_type":"code","source":"def train(model, num_epochs, train_dl, valid_dl):\n    model.to(device)\n    loss_hist_train = [0] * num_epochs\n    accuracy_hist_train = [0] * num_epochs\n    loss_hist_valid = [0] * num_epochs\n    accuracy_hist_valid = [0] * num_epochs\n    start_time = time.time() \n    for epoch in range(num_epochs):\n        \n        model = model.train()\n        for batch_idx, (x_batch, y_batch) in enumerate(train_dl):\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n            # forward\n            logits = model(x_batch)\n            loss = loss_fn(logits, y_batch)\n            _, pred = torch.max(logits, 1)\n            # backward\n            optimizer.zero_grad()\n            loss.backward()\n            # update\n            optimizer.step()\n            # logging\n            loss_hist_train[epoch] += loss.item() * y_batch.size(0)\n            is_correct = ((pred >= 0.5).float() == y_batch).float()\n            accuracy_hist_train[epoch] += is_correct.sum()\n            \n            if not batch_idx % 50:\n                print ('Epoch: %03d/%03d | Batch %03d/%03d | Loss: %.4f' \n                   %(epoch+1, num_epochs, batch_idx, \n                     len(train_dl), loss))\n            \n        loss_hist_train[epoch] /= len(train_dl.dataset)\n        accuracy_hist_train[epoch] /= len(train_dl.dataset)\n        \n        model = model.eval()\n        with torch.no_grad():\n            for x_batch, y_batch in valid_dl:\n                x_batch = x_batch.to(device)\n                y_batch = y_batch.to(device)\n                logits = model(x_batch)\n                loss = loss_fn(logits, y_batch)\n                _, pred = torch.max(logits, 1)\n                loss_hist_valid[epoch] += loss.item() * y_batch.size(0)\n                is_correct = ((pred >= 0.5).float() == y_batch).float()\n                accuracy_hist_valid[epoch] += is_correct.sum()\n            loss_hist_valid[epoch] /= len(valid_dl.dataset)\n            accuracy_hist_valid[epoch] /= len(valid_dl.dataset)\n        print(f'Epoch {epoch+1} accuracy: '\n              f'{accuracy_hist_train[epoch]:.4f} val_accuracy: '\n              f'{accuracy_hist_valid[epoch]:.4f}')\n        print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n    print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n    return loss_hist_train, loss_hist_valid, accuracy_hist_train, accuracy_hist_valid","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:18.138556Z","iopub.execute_input":"2023-09-15T19:50:18.138983Z","iopub.status.idle":"2023-09-15T19:50:18.156930Z","shell.execute_reply.started":"2023-09-15T19:50:18.138927Z","shell.execute_reply":"2023-09-15T19:50:18.155945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = train(model, NUM_EPOCHS, tr_dl, va_dl)\n\n# write history to file\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_from_scratch.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-15T11:55:33.640942Z","iopub.execute_input":"2023-09-15T11:55:33.641313Z","iopub.status.idle":"2023-09-15T11:56:02.833703Z","shell.execute_reply.started":"2023-09-15T11:55:33.641275Z","shell.execute_reply":"2023-09-15T11:56:02.832174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 Visualize the learning curve  <a class=\"anchor\" id=\"35\"></a>","metadata":{}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_from_scratch.png', dpi = my_dpi);","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:02.343820Z","iopub.execute_input":"2023-09-15T19:46:02.344947Z","iopub.status.idle":"2023-09-15T19:46:02.838326Z","shell.execute_reply.started":"2023-09-15T19:46:02.344902Z","shell.execute_reply":"2023-09-15T19:46:02.836291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.6 Make predictions on test set  <a class=\"anchor\" id=\"36\"></a>","metadata":{}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Build a model using AlexNet weights  <a class=\"anchor\" id=\"4\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 4.1 Load pre-trained AlexNet <a class=\"anchor\" id=\"41\"></a>","metadata":{}},{"cell_type":"code","source":"alexnet = models.alexnet(pretrained = True)\nalexnet","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:15.607334Z","iopub.execute_input":"2023-09-15T19:43:15.607869Z","iopub.status.idle":"2023-09-15T19:43:17.566417Z","shell.execute_reply.started":"2023-09-15T19:43:15.607836Z","shell.execute_reply":"2023-09-15T19:43:17.565413Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 Freeze early layers  <a class=\"anchor\" id=\"42\"></a>","metadata":{}},{"cell_type":"code","source":"for param in alexnet.features.parameters():\n    param.requires_grad = False\n\n# for i, clf in enumerate(alexnet.classifier):\n#     if i != 6:\n#         continue\n#     else:\n#         for param in clf.parameters():\n#             param.requires_grad = False\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:43:30.452995Z","iopub.execute_input":"2023-09-15T19:43:30.453373Z","iopub.status.idle":"2023-09-15T19:43:30.459512Z","shell.execute_reply.started":"2023-09-15T19:43:30.453342Z","shell.execute_reply":"2023-09-15T19:43:30.458085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.3 Modify output layer <a class=\"anchor\" id=\"43\"></a>","metadata":{}},{"cell_type":"code","source":"alexnet.classifier[6] = nn.Linear(alexnet.classifier[6].in_features, NUM_CLASSES)","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:55:29.990831Z","iopub.execute_input":"2023-09-15T19:55:29.991424Z","iopub.status.idle":"2023-09-15T19:55:29.997933Z","shell.execute_reply.started":"2023-09-15T19:55:29.991388Z","shell.execute_reply":"2023-09-15T19:55:29.996871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.4 Train model <a class=\"anchor\" id=\"44\"></a>","metadata":{}},{"cell_type":"code","source":"model = alexnet\nmodel.to(device)\n\nloss_fn = F.cross_entropy\noptimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\n\nhistory = train(model, NUM_EPOCHS, tr_dl, va_dl)\nl1 = history[0]\nl2 = history[1]\nl3 = [i.item() for i in history[2]]\nl4 = [i.item() for i in history[3]]\ndf = pd.DataFrame(zip(l1,l2,l3,l4), columns =['Train_loss', 'Validation_loss', 'Train_accuracy', 'Validation accuracy'])\ndf.to_csv('history_transfer_learning.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:21.071120Z","iopub.execute_input":"2023-09-15T19:50:21.071491Z","iopub.status.idle":"2023-09-15T19:50:48.987589Z","shell.execute_reply.started":"2023-09-15T19:50:21.071462Z","shell.execute_reply":"2023-09-15T19:50:48.985868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.5 Visualize the learning curve <a class=\"anchor\" id=\"45\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:46:45.355441Z","iopub.execute_input":"2023-09-15T19:46:45.355826Z","iopub.status.idle":"2023-09-15T19:46:45.362780Z","shell.execute_reply.started":"2023-09-15T19:46:45.355793Z","shell.execute_reply":"2023-09-15T19:46:45.361834Z"}}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history_transfer_learning.png', dpi = my_dpi);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.6 Make prediction on test set <a class=\"anchor\" id=\"46\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-09-15T19:50:48.989266Z","iopub.status.idle":"2023-09-15T19:50:48.990100Z","shell.execute_reply.started":"2023-09-15T19:50:48.989835Z","shell.execute_reply":"2023-09-15T19:50:48.989861Z"}}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=BATCH_SIZE, shuffle=False, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\nlogits = model(x_batch)\n_, pred = torch.max(logits, 1)\npred *= 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}