{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Imports and Notebook setup","metadata":{}},{"cell_type":"code","source":"# library with nice formating\n!pip install rich","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:04.267950Z","iopub.execute_input":"2021-10-26T07:19:04.268499Z","iopub.status.idle":"2021-10-26T07:19:10.716340Z","shell.execute_reply.started":"2021-10-26T07:19:04.268452Z","shell.execute_reply":"2021-10-26T07:19:10.715530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:12.354916Z","iopub.execute_input":"2021-10-26T07:19:12.355198Z","iopub.status.idle":"2021-10-26T07:19:12.417739Z","shell.execute_reply.started":"2021-10-26T07:19:12.355165Z","shell.execute_reply":"2021-10-26T07:19:12.416998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# default_exp resnet\nimport os\n\nif os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') == 'Losthost':\n    from nbdev import *","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:13.421727Z","iopub.execute_input":"2021-10-26T07:19:13.422206Z","iopub.status.idle":"2021-10-26T07:19:13.426252Z","shell.execute_reply.started":"2021-10-26T07:19:13.422170Z","shell.execute_reply":"2021-10-26T07:19:13.425302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\nimport rich\n\nfrom collections import OrderedDict","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:13.853024Z","iopub.execute_input":"2021-10-26T07:19:13.853513Z","iopub.status.idle":"2021-10-26T07:19:13.859185Z","shell.execute_reply.started":"2021-10-26T07:19:13.853479Z","shell.execute_reply":"2021-10-26T07:19:13.858241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti\nimport numpy as np\nimport pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:14.393196Z","iopub.execute_input":"2021-10-26T07:19:14.393837Z","iopub.status.idle":"2021-10-26T07:19:14.397509Z","shell.execute_reply.started":"2021-10-26T07:19:14.393802Z","shell.execute_reply":"2021-10-26T07:19:14.396823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:15.756370Z","iopub.execute_input":"2021-10-26T07:19:15.756914Z","iopub.status.idle":"2021-10-26T07:19:19.962196Z","shell.execute_reply.started":"2021-10-26T07:19:15.756877Z","shell.execute_reply":"2021-10-26T07:19:19.961443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\nfrom torch.utils.data import dataset, dataloader, random_split\n\nfrom torchvision.datasets import CIFAR10\nfrom torchvision.transforms import ToTensor, Lambda, Compose","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:19.964258Z","iopub.execute_input":"2021-10-26T07:19:19.964670Z","iopub.status.idle":"2021-10-26T07:19:20.099694Z","shell.execute_reply.started":"2021-10-26T07:19:19.964637Z","shell.execute_reply":"2021-10-26T07:19:20.098967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ndataset = CIFAR10(root=\"/kaggle/working/data/\",\n                        download=True,\n                        train=True, transform=ToTensor())\ntest_dataset = CIFAR10(root=\"/kaggle/working/data/\",\n                       download=True, train=False,\n                       transform=ToTensor())\n\n# create train, valid datasets\nvalid_size = int(dataset.data.shape[0]*0.1)\ntrain_size = dataset.data.shape[0] - valid_size\n\ntrain_dataset, valid_dataset = random_split(dataset, [train_size, valid_size])","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:20.100960Z","iopub.execute_input":"2021-10-26T07:19:20.101312Z","iopub.status.idle":"2021-10-26T07:19:28.396953Z","shell.execute_reply.started":"2021-10-26T07:19:20.101277Z","shell.execute_reply":"2021-10-26T07:19:28.396167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\nconfig = OrderedDict()\nconfig['batch_size'] = 128","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:28.398775Z","iopub.execute_input":"2021-10-26T07:19:28.399041Z","iopub.status.idle":"2021-10-26T07:19:28.404686Z","shell.execute_reply.started":"2021-10-26T07:19:28.399014Z","shell.execute_reply":"2021-10-26T07:19:28.403821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ntrain_dataloader = dataloader.DataLoader(train_dataset, batch_size=config['batch_size'])\nvalid_dataloader = dataloader.DataLoader(valid_dataset, batch_size=config['batch_size'])\ntest_dataloader = dataloader.DataLoader(test_dataset, batch_size=config['batch_size'])","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:28.406083Z","iopub.execute_input":"2021-10-26T07:19:28.406953Z","iopub.status.idle":"2021-10-26T07:19:28.413448Z","shell.execute_reply.started":"2021-10-26T07:19:28.406914Z","shell.execute_reply":"2021-10-26T07:19:28.412704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hide\nfor x, y in train_dataloader:\n    rich.print(x.max(), x.min())\n    rich.print(y)\n    break","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:30.433669Z","iopub.execute_input":"2021-10-26T07:19:30.433939Z","iopub.status.idle":"2021-10-26T07:19:30.546706Z","shell.execute_reply.started":"2021-10-26T07:19:30.433909Z","shell.execute_reply":"2021-10-26T07:19:30.545955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ResNet Block","metadata":{}},{"cell_type":"code","source":"# export\nclass ResNetBlock(nn.Module):\n    \n    def __init__(self, in_channels: int = 32, num_filters: int = 64):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels=in_channels,\n                               out_channels=num_filters,\n                               kernel_size=3, padding=1)\n        self.bn1 = nn.BatchNorm2d(num_filters)\n        self.relu1 = nn.ReLU()\n        self.conv2 = nn.Conv2d(in_channels=num_filters,\n                               out_channels=num_filters, \n                               kernel_size=3, padding=1)\n        self.bn2 = nn.BatchNorm2d(num_filters)\n        self.relu2 = nn.ReLU()\n    \n    def forward(self, x):\n        z = self.conv1(x)\n        z = self.relu1(z)\n        z = self.bn1(z)\n        z = self.conv2(z)\n        y = self.relu2(z + x)\n        return self.bn2(y)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:33.283641Z","iopub.execute_input":"2021-10-26T07:19:33.284232Z","iopub.status.idle":"2021-10-26T07:19:33.292897Z","shell.execute_reply.started":"2021-10-26T07:19:33.284193Z","shell.execute_reply":"2021-10-26T07:19:33.292118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that `ResNetBlock` has same input and output sizes","metadata":{}},{"cell_type":"code","source":"rblock = ResNetBlock(in_channels=64)\nx = torch.randint(0, 100, size=(128, 64, 32, 32), dtype=torch.float32)\ny = rblock(x)\nassert x.shape == y.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:19:35.031380Z","iopub.execute_input":"2021-10-26T07:19:35.031650Z","iopub.status.idle":"2021-10-26T07:19:35.580817Z","shell.execute_reply.started":"2021-10-26T07:19:35.031621Z","shell.execute_reply":"2021-10-26T07:19:35.580115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`ResNetChangeBlock` implements the ResNet with skip connections when the input and output have different shape","metadata":{}},{"cell_type":"code","source":"# export\nclass ResNetChangeBlock(nn.Module):\n    \n    def __init__(self, in_channels: int = 32, num_filters: int = 64):\n        super().__init__()\n        self.in_channels = in_channels\n        self.out_channels = num_filters\n        self.conv1 = nn.Conv2d(in_channels=in_channels,\n                               out_channels=num_filters,\n                               stride=2,\n                               kernel_size=3, padding=1)\n        self.bn1 = nn.BatchNorm2d(num_filters)\n        self.relu1 = nn.ReLU()\n        self.conv2 = nn.Conv2d(in_channels=num_filters,\n                               out_channels=num_filters, \n                               kernel_size=3, padding=1)\n        self.bn2 = nn.BatchNorm2d(num_filters)\n        self.relu2 = nn.ReLU()\n    \n    def forward(self, x):\n        n, c, w, h = x.shape\n        z = self.conv1(x)\n        z = self.relu1(z)\n        z = self.bn1(z)\n        z = self.conv2(z)\n        self.zero_identity = nn.Parameter(\n            torch.zeros(n, self.out_channels-self.in_channels,\n                        int(w//2), int(h//2)),\n            requires_grad=False\n        )\n        x_expand = F.pad(x[:, :, :int(w//2), :int(h//2)], pad=(\n            0, 0, 0, 0, self.out_channels-self.in_channels, 0, 0, 0))\n        y = self.relu2(z + x_expand)\n        return self.bn2(y)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:54:16.579392Z","iopub.execute_input":"2021-10-26T07:54:16.579916Z","iopub.status.idle":"2021-10-26T07:54:16.590490Z","shell.execute_reply.started":"2021-10-26T07:54:16.579881Z","shell.execute_reply":"2021-10-26T07:54:16.589799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check the padding operation works as intended, the pad dimensions are left-to-right last-to-first dimension","metadata":{}},{"cell_type":"code","source":"x = torch.ones([128, 16, 16, 16])\ny = torch.ones([128, 32, 16, 16])\nz = F.pad(x, (0, 0, 0, 0, 16, 0, 0, 0))\nassert z.shape == y.shape","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:51:51.604937Z","iopub.execute_input":"2021-10-26T07:51:51.605622Z","iopub.status.idle":"2021-10-26T07:51:51.612614Z","shell.execute_reply.started":"2021-10-26T07:51:51.605585Z","shell.execute_reply":"2021-10-26T07:51:51.611910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check that `ResNetChangeBlock` has half receptor field","metadata":{}},{"cell_type":"code","source":"rblock2 = ResNetChangeBlock(in_channels=64, num_filters=128)\nx = torch.randint(0, 100, size=(128, 64, 32, 32), dtype=torch.float32)\ny = rblock2(x)\nassert y.shape == (128, 128, 16, 16)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:54:41.554382Z","iopub.execute_input":"2021-10-26T07:54:41.554852Z","iopub.status.idle":"2021-10-26T07:54:41.928432Z","shell.execute_reply.started":"2021-10-26T07:54:41.554821Z","shell.execute_reply":"2021-10-26T07:54:41.927621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`ResNet` with variable `6n+2` number of layers and skip connections `2n` skip connections, Similar to CIFAR10 network from the [resnet paper](https://arxiv.org/abs/1512.03385#).","metadata":{}},{"cell_type":"code","source":"# export\nclass ResNet(nn.Module):\n    \n    def __init__(self, n: int = 3, num_classes: int = 10):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels=3, out_channels=16,\n                               kernel_size=3, padding=1)\n        self.resblock1_list = nn.ModuleList([\n            ResNetBlock(in_channels=16, num_filters=16) for _ in range(n)\n        ])\n        self.resblock2_list = nn.ModuleList([\n                ResNetChangeBlock(in_channels=16, num_filters=32)\n            ] + [\n            ResNetBlock(in_channels=32, num_filters=32) for _ in range(n-1)\n        ])\n        self.resblock3_list = nn.ModuleList([\n            ResNetChangeBlock(in_channels=32, num_filters=64)\n        ] + [\n            ResNetBlock(in_channels=64, num_filters=64) for _ in range(n-1)\n        ])\n        self.linear1 = nn.Linear(in_features=64,\n                                 out_features=num_classes)\n        \n    def forward(self, x):\n        x = self.conv1(x)\n        for layer1 in self.resblock1_list:\n            x = layer1(x)\n        for layer2 in self.resblock2_list:\n            x = layer2(x)\n        for layer3 in self.resblock3_list:\n            x = layer3(x)\n        x = torch.mean(x, dim=[2, 3])\n        x = self.linear1(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:07.015973Z","iopub.execute_input":"2021-10-26T07:56:07.016600Z","iopub.status.idle":"2021-10-26T07:56:07.026822Z","shell.execute_reply.started":"2021-10-26T07:56:07.016560Z","shell.execute_reply":"2021-10-26T07:56:07.025994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Check `ResNet` has last dimension as `num_classes`","metadata":{}},{"cell_type":"code","source":"n = 3\nnum_classes = 10\nbatch_size = 128\nres_net = ResNet(n=n, num_classes=num_classes)\nx = torch.randint(0, 100, size=(batch_size, 3, 32, 32), dtype=torch.float32)\ny = res_net(x)\nassert y.shape == (batch_size, num_classes)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:08.764653Z","iopub.execute_input":"2021-10-26T07:56:08.765377Z","iopub.status.idle":"2021-10-26T07:56:09.140354Z","shell.execute_reply.started":"2021-10-26T07:56:08.765328Z","shell.execute_reply":"2021-10-26T07:56:09.139604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define Train and Eval loop","metadata":{}},{"cell_type":"code","source":"# exporti train\nconfig['device'] = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"device: {config['device']}\")\n\n# define the network\nconfig['n'] = 3\nnum_classes = 10\nres_net = ResNet(n=config['n'], num_classes=num_classes)\nres_net = res_net.to(config['device'])\n\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(params=res_net.parameters())","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:10.436618Z","iopub.execute_input":"2021-10-26T07:56:10.437174Z","iopub.status.idle":"2021-10-26T07:56:10.460250Z","shell.execute_reply.started":"2021-10-26T07:56:10.437138Z","shell.execute_reply":"2021-10-26T07:56:10.459540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rich.print(res_net)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:43.431002Z","iopub.execute_input":"2021-10-26T07:56:43.431425Z","iopub.status.idle":"2021-10-26T07:56:43.505662Z","shell.execute_reply.started":"2021-10-26T07:56:43.431389Z","shell.execute_reply":"2021-10-26T07:56:43.504918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ndef train_fn(model: nn.Module, \n             train_dataloader: dataloader.DataLoader,\n             loss_fn, optimizer, epoch: int, device: str):\n    \n    model.train()\n    \n    for batch_index, (X, y) in tqdm(enumerate(train_dataloader),\n                                    total=len(train_dataloader.dataset.indices)//config['batch_size']):\n        \n        X_data, y_data = X.to(device), y.to(device)\n        y_pred = model(X_data)\n        loss = loss_fn(y_pred, y_data)\n        \n        # backprop\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    loss = loss.item()\n    accu = torch.argmax(y_pred, dim=-1) == y_data\n    accu = sum(accu)/len(accu)\n    accu = accu.item()\n    print(\"loss=\"+str(loss), end=' ')\n    print(\"accu=\"+str(accu), end=' ')\n","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:47.016639Z","iopub.execute_input":"2021-10-26T07:56:47.016949Z","iopub.status.idle":"2021-10-26T07:56:47.030045Z","shell.execute_reply.started":"2021-10-26T07:56:47.016913Z","shell.execute_reply":"2021-10-26T07:56:47.029060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ndef test_fn(model: nn.Module,\n            test_dataloader: dataloader.DataLoader,\n            epoch: int, device: str, dataset: str):\n    \n    loss = 0\n    results = []\n    model.eval()\n    with torch.no_grad():\n        for batch_index, (X, y) in tqdm(enumerate(test_dataloader)):\n            X_data, y_data = X.to(device), y.to(device)\n            y_pred = model(X_data)\n            loss += loss_fn(y_pred, y_data).item()\n            results.extend(torch.argmax(y_pred, dim=-1) == y_data)\n    \n    test_accu = sum(results)/len(results)\n    test_accu = test_accu.item()\n    test_loss = loss/len(results)\n    print(dataset + \"_loss: \" + str(test_loss), end=' ')\n    print(dataset + \"_accu: \" + str(test_accu), end=' ')\n    return False","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:47.400974Z","iopub.execute_input":"2021-10-26T07:56:47.401548Z","iopub.status.idle":"2021-10-26T07:56:47.409262Z","shell.execute_reply.started":"2021-10-26T07:56:47.401512Z","shell.execute_reply":"2021-10-26T07:56:47.408201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train and Track the Model","metadata":{}},{"cell_type":"code","source":"# exporti train\n\nconfig['num_epochs'] = 25\n\nfor epoch in tqdm(range(config['num_epochs'])):\n    print(\"epoch: \" + str(epoch), end=' ')\n    train_fn(res_net, train_dataloader, loss_fn, optimizer, epoch, config['device'])\n    test_fn(res_net, valid_dataloader, epoch, config['device'], dataset=\"valid\")    \n    print()","metadata":{"execution":{"iopub.status.busy":"2021-10-26T07:56:52.377576Z","iopub.execute_input":"2021-10-26T07:56:52.378135Z","iopub.status.idle":"2021-10-26T08:00:03.115961Z","shell.execute_reply.started":"2021-10-26T07:56:52.378080Z","shell.execute_reply":"2021-10-26T08:00:03.114908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test and Save the network","metadata":{}},{"cell_type":"code","source":"# exporti predict\ndef predict_fn(model, test_dataloader, device):\n\n    y_pred_list = []\n    with torch.no_grad():\n        for batch_index, (X, y) in enumerate(test_dataloader):\n            X_data, y_data = X.to(device), y.to(device)\n            y_pred = model(X_data)\n            y_pred_list.append(y_pred)\n    return torch.cat(y_pred_list)","metadata":{"execution":{"iopub.status.busy":"2021-10-26T08:01:38.522811Z","iopub.execute_input":"2021-10-26T08:01:38.523087Z","iopub.status.idle":"2021-10-26T08:01:38.528806Z","shell.execute_reply.started":"2021-10-26T08:01:38.523057Z","shell.execute_reply":"2021-10-26T08:01:38.528131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ntest_fn(res_net, test_dataloader, epoch, config['device'], dataset=\"test\")","metadata":{"execution":{"iopub.status.busy":"2021-10-26T08:01:39.168880Z","iopub.execute_input":"2021-10-26T08:01:39.169404Z","iopub.status.idle":"2021-10-26T08:01:40.831826Z","shell.execute_reply.started":"2021-10-26T08:01:39.169369Z","shell.execute_reply":"2021-10-26T08:01:40.831009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exporti train\ntorch.save(res_net.state_dict(), \"resnet_cifar10_n3_network.pth\")","metadata":{"execution":{"iopub.status.busy":"2021-10-26T08:01:45.676905Z","iopub.execute_input":"2021-10-26T08:01:45.677177Z","iopub.status.idle":"2021-10-26T08:01:45.699608Z","shell.execute_reply.started":"2021-10-26T08:01:45.677147Z","shell.execute_reply":"2021-10-26T08:01:45.698945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Conclusion and Future work","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}