{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-07-03T01:04:42.649408Z","iopub.execute_input":"2023-07-03T01:04:42.650464Z","iopub.status.idle":"2023-07-03T01:04:42.668437Z","shell.execute_reply.started":"2023-07-03T01:04:42.650412Z","shell.execute_reply":"2023-07-03T01:04:42.666835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Video walkthrough:\n- [link](https://youtube.com/shorts/1is9kx8G0PA?feature=share)\n\n## Contents:\n* [Import packages](#1)\n* [Preprocess image](#2)\n    * [Unzip](#21)    \n    * [Show image examples](#22)\n    * [Check image sizes](#23)    \n    * [Augmentation while loading](#24)\n    * [Check augmented images](#25)\n* [Build a CNN model](#3)\n    * [Define model](#31)\n    * [Visualize using Torchviz](#33)\n    * [Choose loss function, optimizer](#33)\n    * [Train model](#34)\n    * [Visualize the learning curve](#35)\n* [Make Predictions on test set](#4)","metadata":{}},{"cell_type":"markdown","source":"## 1. Import packages <a class=\"anchor\" id=\"1\"></a>","metadata":{}},{"cell_type":"code","source":"!pip install torchviz","metadata":{"_kg_hide-output":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2023-07-03T01:04:43.764960Z","iopub.execute_input":"2023-07-03T01:04:43.765891Z","iopub.status.idle":"2023-07-03T01:04:59.353753Z","shell.execute_reply.started":"2023-07-03T01:04:43.765850Z","shell.execute_reply":"2023-07-03T01:04:59.352477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import zipfile\nimport glob\nimport time\nimport multiprocessing\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader, Subset\nfrom torchviz import make_dot\nSEED = 1","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:04:59.357703Z","iopub.execute_input":"2023-07-03T01:04:59.358080Z","iopub.status.idle":"2023-07-03T01:05:03.714312Z","shell.execute_reply.started":"2023-07-03T01:04:59.358040Z","shell.execute_reply":"2023-07-03T01:05:03.713239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Preprocess image <a class=\"anchor\" id=\"2\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 2.1 Unzip <a class=\"anchor\" id=\"21\"></a>","metadata":{}},{"cell_type":"code","source":"base_dir = '../input/dogs-vs-cats-redux-kernels-edition'\ntrain_dir = '../data/train'\ntest_dir = '../data/test'\n\nwith zipfile.ZipFile(os.path.join(base_dir, 'train.zip')) as train_zip:\n    train_zip.extractall('../data')\n    \nwith zipfile.ZipFile(os.path.join(base_dir, 'test.zip')) as test_zip:\n    test_zip.extractall('../data')\n    \ntrain_list = glob.glob(os.path.join(train_dir,'*.jpg'))\ntest_list = glob.glob(os.path.join(test_dir, '*.jpg'))\n\nlen(train_list), len(test_list)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:03.715684Z","iopub.execute_input":"2023-07-03T01:05:03.716978Z","iopub.status.idle":"2023-07-03T01:05:21.228633Z","shell.execute_reply.started":"2023-07-03T01:05:03.716943Z","shell.execute_reply":"2023-07-03T01:05:21.227633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Show training image examples <a class=\"anchor\" id=\"22\"></a>","metadata":{}},{"cell_type":"code","source":"random_idx = np.random.randint(1,25000,size=10)\n\nfig = plt.figure(figsize = (15,6))\ni=1\nfor idx in random_idx:\n    ax = fig.add_subplot(2,5,i)\n    img = Image.open(train_list[idx])\n    img_path = train_list[idx]\n    label = img_path.split('/')[-1].split('.')[0]\n    plt.imshow(img)\n    plt.title(label, size=15)\n    i+=1\n    \nplt.tight_layout()\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:21.234373Z","iopub.execute_input":"2023-07-03T01:05:21.234837Z","iopub.status.idle":"2023-07-03T01:05:23.918880Z","shell.execute_reply.started":"2023-07-03T01:05:21.234802Z","shell.execute_reply":"2023-07-03T01:05:23.917737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Check the original sizes of the training images and channels <a class=\"anchor\" id=\"23\"></a>","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:14:14.711891Z","iopub.execute_input":"2023-07-02T22:14:14.712268Z","iopub.status.idle":"2023-07-02T22:14:14.717015Z","shell.execute_reply.started":"2023-07-02T22:14:14.712238Z","shell.execute_reply":"2023-07-02T22:14:14.715667Z"}}},{"cell_type":"code","source":"h_list = []\nw_list = []\nband_set = set()\nfor train_img in train_list:\n    img = Image.open(train_img)\n    w, h = img.size\n    h_list.append(h)\n    w_list.append(w)\n    band_set.add(img.getbands())\n    \nw_med, h_med = np.percentile(w_list, q = 80), np.percentile(h_list, q = 80)\nprint('80 percentile width of all the training images = ', w_med, \n      '\\n80 percentile height of all the training images =', h_med)\nprint('channels of the training images:', band_set)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:30.424002Z","iopub.execute_input":"2023-07-03T01:05:30.424424Z","iopub.status.idle":"2023-07-03T01:05:33.275301Z","shell.execute_reply.started":"2023-07-03T01:05:30.424392Z","shell.execute_reply":"2023-07-03T01:05:33.274082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Image augmentation during data loading <a class=\"anchor\" id=\"24\"></a>","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\ntorch.manual_seed(SEED)\nif device =='cuda':\n    torch.cuda.manual_seed_all(SEED)\nprint(\"device:\", device)","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:35.104083Z","iopub.execute_input":"2023-07-03T01:05:35.104892Z","iopub.status.idle":"2023-07-03T01:05:35.186164Z","shell.execute_reply.started":"2023-07-03T01:05:35.104857Z","shell.execute_reply":"2023-07-03T01:05:35.185071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_transforms = {\n    'train': transforms.Compose([\n        transforms.RandomResizedCrop([512,512]),\n        transforms.RandomHorizontalFlip(),\n        transforms.Resize([256,256]),\n        transforms.ToTensor(),\n    ]),\n    'test': transforms.Compose([\n        transforms.Resize([256,256]),\n        transforms.ToTensor()\n    ])\n}","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:36.392915Z","iopub.execute_input":"2023-07-03T01:05:36.394376Z","iopub.status.idle":"2023-07-03T01:05:36.403749Z","shell.execute_reply.started":"2023-07-03T01:05:36.394321Z","shell.execute_reply":"2023-07-03T01:05:36.400976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class dataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.file_list = file_list\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.file_list)\n\n    def __getitem__(self, idx):\n        img_path = self.file_list[idx]\n        img = Image.open(img_path)\n        if self.transform is not None:\n            img = self.transform(img)\n        label = img_path.split('/')[-1].split('.')[0]\n        if label == 'dog':\n            label = 1\n        elif label == 'cat':\n            label = 0\n            \n        return img, label","metadata":{"execution":{"iopub.status.busy":"2023-07-03T01:05:40.121488Z","iopub.execute_input":"2023-07-03T01:05:40.121854Z","iopub.status.idle":"2023-07-03T01:05:40.129953Z","shell.execute_reply.started":"2023-07-03T01:05:40.121825Z","shell.execute_reply":"2023-07-03T01:05:40.128950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trlist, valist = train_test_split(train_list, test_size=0.1)\n\ntr_ds = dataset(trlist, transform=data_transforms['train'])\nva_ds = dataset(valist, transform=data_transforms['test'])\nte_ds = dataset(test_list, transform=data_transforms['test'])\n\n# tr_ds = Subset(tr_ds, torch.arange(1000))\n# va_ds = Subset(va_ds, torch.arange(100))\n\nbatch_size = 32\ntorch.manual_seed(SEED)\nnum_workers = multiprocessing.cpu_count()\ntr_dl = DataLoader(tr_ds, batch_size=batch_size, shuffle=True, num_workers = num_workers, pin_memory = True)\nva_dl = DataLoader(va_ds, batch_size=batch_size, shuffle=False, num_workers = num_workers, pin_memory = True)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:23:59.119938Z","iopub.execute_input":"2023-07-02T22:23:59.120840Z","iopub.status.idle":"2023-07-02T22:23:59.140510Z","shell.execute_reply.started":"2023-07-02T22:23:59.120766Z","shell.execute_reply":"2023-07-02T22:23:59.139674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.5 Check image augmentation <a class=\"anchor\" id=\"25\"></a>","metadata":{}},{"cell_type":"code","source":"tmp_dl = DataLoader(tr_ds, batch_size=2)\nfig = plt.figure(figsize=(15, 6))\nnum_epochs = 5\nfor j in range(num_epochs):\n    img_batch, label_batch = next(iter(tmp_dl))\n    img = img_batch[0]\n    ax = fig.add_subplot(2, 5, j + 1)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.set_title(f'Epoch {j}:', size=15)\n    ax.imshow(img.permute(1, 2, 0))\n    \n    img = img_batch[1]\n    ax = fig.add_subplot(2, 5, j + 6)\n    ax.set_xticks([])\n    ax.set_yticks([])\n    ax.imshow(img.permute(1, 2, 0))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:25:39.051355Z","iopub.execute_input":"2023-07-02T22:25:39.051719Z","iopub.status.idle":"2023-07-02T22:25:39.462424Z","shell.execute_reply.started":"2023-07-02T22:25:39.051692Z","shell.execute_reply":"2023-07-02T22:25:39.461598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Build a CNN model <a class=\"anchor\" id=\"3\"></a>","metadata":{}},{"cell_type":"markdown","source":"### 3.1 Define model <a class=\"anchor\" id=\"31\"></a>","metadata":{}},{"cell_type":"code","source":"model = nn.Sequential()\nkernel_size = 3\npadding = (kernel_size - 1) // 2\nmodel.add_module('conv1',\n                nn.Conv2d(in_channels = 3,\n                         out_channels = 32,\n                         kernel_size = kernel_size,\n                         padding = padding))\nmodel.add_module('relu1', nn.ReLU())\nmodel.add_module('pool1', nn.MaxPool2d(kernel_size = 2))\nmodel.add_module('dropout1', nn.Dropout(p = 0.5))\n\nmodel.add_module('conv2', nn.Conv2d(in_channels = 32, \n                                    out_channels = 64,\n                                    kernel_size = kernel_size, \n                                    padding = padding\n                                   )\n                )\nmodel.add_module('relu2', nn.ReLU())\nmodel.add_module('pool2', nn.MaxPool2d(kernel_size = 2))\nmodel.add_module('dropout2', nn.Dropout(p = 0.5))\n\nmodel.add_module('conv3', nn.Conv2d(in_channels = 64, \n                                    out_channels = 128,\n                                    kernel_size = kernel_size, \n                                    padding = padding\n                                   )\n                )\nmodel.add_module('relu3', nn.ReLU())\nmodel.add_module('pool3', nn.MaxPool2d(kernel_size = 2))\nmodel.add_module('dropout3', nn.Dropout(p = 0.5))\n\nmodel.add_module('conv4', nn.Conv2d(in_channels = 128, \n                                    out_channels = 256,\n                                    kernel_size = kernel_size, \n                                    padding = padding\n                                   )\n                )\nmodel.add_module('relu4', nn.ReLU())\nmodel.add_module('pool4', nn.MaxPool2d(kernel_size = 2))\nmodel.add_module('dropout4', nn.Dropout(p = 0.5))\n\nmodel.add_module('conv5', nn.Conv2d(in_channels = 256, \n                                    out_channels = 512,\n                                    kernel_size = kernel_size, \n                                    padding = padding\n                                   )\n                )\nmodel.add_module('relu5', nn.ReLU())\n# global average pooling layer\nmodel.add_module('pool5', nn.AvgPool2d(kernel_size = 16))\nmodel.add_module('flatten', nn.Flatten())\nmodel.add_module('fc', nn.Linear(512,1))\nmodel.add_module('sigmoid', nn.Sigmoid())\n\nmodel","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:31:54.025650Z","iopub.execute_input":"2023-07-02T22:31:54.026057Z","iopub.status.idle":"2023-07-02T22:31:54.070336Z","shell.execute_reply.started":"2023-07-02T22:31:54.026015Z","shell.execute_reply":"2023-07-02T22:31:54.069160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 Show NN using Torchviz <a class=\"anchor\" id=\"32\"></a>","metadata":{}},{"cell_type":"code","source":"x = torch.ones([1, 3, 256, 256])\ny = model(x)\nmake_dot(y, params=dict(list(model.named_parameters())))#.render(\"attached_model\", format=\"png\")","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:35:47.268195Z","iopub.execute_input":"2023-07-02T22:35:47.268549Z","iopub.status.idle":"2023-07-02T22:35:47.383989Z","shell.execute_reply.started":"2023-07-02T22:35:47.268522Z","shell.execute_reply":"2023-07-02T22:35:47.383020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.3 Choose loss function, optimizer <a class=\"anchor\" id=\"33\"></a>","metadata":{}},{"cell_type":"code","source":"loss_fn = nn.BCELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr = 0.001)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:41:42.312486Z","iopub.execute_input":"2023-07-02T22:41:42.312870Z","iopub.status.idle":"2023-07-02T22:41:42.319795Z","shell.execute_reply.started":"2023-07-02T22:41:42.312836Z","shell.execute_reply":"2023-07-02T22:41:42.318832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Train model <a class=\"anchor\" id=\"34\"></a>","metadata":{}},{"cell_type":"code","source":"def train(model, num_epochs, train_dl, valid_dl):\n    model.to(device)\n    loss_hist_train = [0] * num_epochs\n    accuracy_hist_train = [0] * num_epochs\n    loss_hist_valid = [0] * num_epochs\n    accuracy_hist_valid = [0] * num_epochs\n    start_time = time.time() \n    for epoch in range(num_epochs):\n        \n        model = model.train()\n        for batch_idx, (x_batch, y_batch) in enumerate(train_dl):\n            x_batch = x_batch.to(device)\n            y_batch = y_batch.to(device)\n            pred = model(x_batch)[:, 0]\n            loss = loss_fn(pred, y_batch.float())\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            loss_hist_train[epoch] += loss.item() * y_batch.size(0)\n            is_correct = ((pred >= 0.5).float() == y_batch).float()\n            accuracy_hist_train[epoch] += is_correct.sum()\n            \n            if not batch_idx % 50:\n                print ('Epoch: %03d/%03d | Batch %03d/%03d | Loss: %.4f' \n                   %(epoch+1, num_epochs, batch_idx, \n                     len(train_dl), loss))\n            \n        loss_hist_train[epoch] /= len(train_dl.dataset)\n        accuracy_hist_train[epoch] /= len(train_dl.dataset)\n        \n        model = model.eval()\n        with torch.no_grad():\n            for x_batch, y_batch in valid_dl:\n                x_batch = x_batch.to(device)\n                y_batch = y_batch.to(device)\n                pred = model(x_batch)[:, 0]\n                loss = loss_fn(pred, y_batch.float())\n                loss_hist_valid[epoch] += loss.item() * y_batch.size(0)\n                is_correct = ((pred >= 0.5).float() == y_batch).float()\n                accuracy_hist_valid[epoch] += is_correct.sum()\n            loss_hist_valid[epoch] /= len(valid_dl.dataset)\n            accuracy_hist_valid[epoch] /= len(valid_dl.dataset)\n        print(f'Epoch {epoch+1} accuracy: '\n              f'{accuracy_hist_train[epoch]:.4f} val_accuracy: '\n              f'{accuracy_hist_valid[epoch]:.4f}')\n        print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n    print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))\n    return loss_hist_train, loss_hist_valid, accuracy_hist_train, accuracy_hist_valid","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:42:15.703015Z","iopub.execute_input":"2023-07-02T22:42:15.703574Z","iopub.status.idle":"2023-07-02T22:42:15.717354Z","shell.execute_reply.started":"2023-07-02T22:42:15.703542Z","shell.execute_reply":"2023-07-02T22:42:15.716156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.manual_seed(SEED)\nnum_epochs = 60\nhistory = train(model, num_epochs,\n               tr_dl, va_dl)","metadata":{"execution":{"iopub.status.busy":"2023-07-02T22:43:07.849957Z","iopub.execute_input":"2023-07-02T22:43:07.850678Z","iopub.status.idle":"2023-07-02T22:50:24.298905Z","shell.execute_reply.started":"2023-07-02T22:43:07.850645Z","shell.execute_reply":"2023-07-02T22:50:24.297100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 Visualize the learning curve  <a class=\"anchor\" id=\"34\"></a>","metadata":{}},{"cell_type":"code","source":"my_dpi = 100\nfig, ax = plt.subplots(nrows = 2, ncols = 1, figsize=(540/my_dpi, 720/my_dpi), dpi=my_dpi)\nax[0].plot(history[0], lw=2)\nax[0].plot(history[1], lw=2)\nax[0].legend(['Train loss', 'Validation loss'], fontsize=12)\nax[0].set_xlabel('Epochs', size=12)\nax[1].plot([i.cpu() for i in history[2]], lw=2)\nax[1].plot([i.cpu() for i in history[3]], lw=2)\nax[1].legend(['Train acc.', 'Validation acc.'], fontsize=12)\nax[1].set_xlabel('Epochs', size=12);\nplt.tight_layout()\nplt.savefig(f'model_train_history.png', dpi = my_dpi);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Make predictions on test set  <a class=\"anchor\" id=\"4\"></a>","metadata":{}},{"cell_type":"code","source":"model.eval()\nte_dl = DataLoader(te_ds, batch_size=batch_size, shuffle=True, num_workers = num_workers, pin_memory = True)\n\nwith torch.no_grad():\n    for x_batch, y_batch in te_dl:\n        break\n        \nx_batch = x_batch.to(device)\npred = model(x_batch)[:, 0] * 100\n\nfig = plt.figure(figsize=(15, 7))\nfor j in range(10, 20):\n    ax = fig.add_subplot(2, 5, j-10+1)\n    ax.set_xticks([]); ax.set_yticks([])\n    ax.imshow(x_batch[j].cpu().permute(1, 2, 0))\n\n    ax.text(\n        0.5, -0.15,\n        f'Pr(dog)={pred[j]:.0f}%',\n        size=16,\n        horizontalalignment='center',\n        verticalalignment='center',\n        transform=ax.transAxes\n    )\nplt.show()","metadata":{},"execution_count":null,"outputs":[]}]}