{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction.\n\n1. Loading dataset (CIFAR10) - Transforming images\n2. VGG-16 with modification to network head\n3. Using pre-trained models\n4. Storing intermediate models\n5. Resnet\n6. Inception v3 <cr>\n\nMost of the concepts explained in this Notebook are learnt from course [Deep Learning](https://padhai.onefourthlabs.in/courses/dl-feb-2019)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"! pip3 install torch torchvision","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-01T15:28:11.584419Z","iopub.execute_input":"2023-05-01T15:28:11.584699Z","iopub.status.idle":"2023-05-01T15:28:24.497507Z","shell.execute_reply.started":"2023-05-01T15:28:11.584672Z","shell.execute_reply":"2023-05-01T15:28:24.496274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\n\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nimport torch.nn as nn\nimport torch.optim as optim","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:24.500507Z","iopub.execute_input":"2023-05-01T15:28:24.500937Z","iopub.status.idle":"2023-05-01T15:28:27.067153Z","shell.execute_reply.started":"2023-05-01T15:28:24.500874Z","shell.execute_reply":"2023-05-01T15:28:27.066044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:27.068504Z","iopub.execute_input":"2023-05-01T15:28:27.069585Z","iopub.status.idle":"2023-05-01T15:28:27.132634Z","shell.execute_reply.started":"2023-05-01T15:28:27.069536Z","shell.execute_reply":"2023-05-01T15:28:27.131417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset, transforms, and visualisation","metadata":{}},{"cell_type":"code","source":"# Data Augumentation. \n\ntransform_train = transforms.Compose([\n    transforms.RandomResizedCrop(224),  # crop image at random and resizes it to 224.\n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])\n\ntransform_test = transforms.Compose([\n    transforms.RandomResizedCrop(224), \n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:27.136371Z","iopub.execute_input":"2023-05-01T15:28:27.137329Z","iopub.status.idle":"2023-05-01T15:28:27.145944Z","shell.execute_reply.started":"2023-05-01T15:28:27.137283Z","shell.execute_reply":"2023-05-01T15:28:27.144967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = torchvision.datasets.CIFAR10(root='./data', train=True, \n                                        download=True, \n                                        transform=transform_train)\ntestset = torchvision.datasets.CIFAR10(root='./data', train=False, \n                                        download=True, \n                                        transform=transform_test)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-01T15:28:27.151678Z","iopub.execute_input":"2023-05-01T15:28:27.153115Z","iopub.status.idle":"2023-05-01T15:28:34.178628Z","shell.execute_reply.started":"2023-05-01T15:28:27.153076Z","shell.execute_reply":"2023-05-01T15:28:34.177600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_classes = 10\nbatch_size = 4\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.180032Z","iopub.execute_input":"2023-05-01T15:28:34.180686Z","iopub.status.idle":"2023-05-01T15:28:34.186502Z","shell.execute_reply.started":"2023-05-01T15:28:34.180643Z","shell.execute_reply":"2023-05-01T15:28:34.185439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataiter = iter(trainloader)\nimages, labels = next(dataiter)\n\nprint(images.shape)\n\nprint(images[1].shape)\nprint(labels[1].item())","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.187932Z","iopub.execute_input":"2023-05-01T15:28:34.188544Z","iopub.status.idle":"2023-05-01T15:28:34.325679Z","shell.execute_reply.started":"2023-05-01T15:28:34.188504Z","shell.execute_reply":"2023-05-01T15:28:34.324586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imshow(img, title):\n    npimg = img.numpy() / 2 + 0.5 # converting the image to to numpy and un-normalise it.\n    plt.figure(figsize=(batch_size, 1))\n    plt.axis('off')\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.title(title)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.327206Z","iopub.execute_input":"2023-05-01T15:28:34.327659Z","iopub.status.idle":"2023-05-01T15:28:34.333901Z","shell.execute_reply.started":"2023-05-01T15:28:34.327621Z","shell.execute_reply":"2023-05-01T15:28:34.332589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_batch_images(dataloader):\n    images, labels = next(iter(dataloader))\n    img = torchvision.utils.make_grid(images)\n    imshow(img, title=[str(x.item()) for x in labels])","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.335609Z","iopub.execute_input":"2023-05-01T15:28:34.336418Z","iopub.status.idle":"2023-05-01T15:28:34.345223Z","shell.execute_reply.started":"2023-05-01T15:28:34.336381Z","shell.execute_reply":"2023-05-01T15:28:34.344108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(4):\n    show_batch_images(trainloader)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.350113Z","iopub.execute_input":"2023-05-01T15:28:34.350391Z","iopub.status.idle":"2023-05-01T15:28:34.926382Z","shell.execute_reply.started":"2023-05-01T15:28:34.350366Z","shell.execute_reply":"2023-05-01T15:28:34.925345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  VGG-16\nhttps://pytorch.org/vision/main/models/vgg.html","metadata":{}},{"cell_type":"code","source":"from torchvision import models","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.927742Z","iopub.execute_input":"2023-05-01T15:28:34.928359Z","iopub.status.idle":"2023-05-01T15:28:34.934734Z","shell.execute_reply.started":"2023-05-01T15:28:34.928319Z","shell.execute_reply":"2023-05-01T15:28:34.933280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = models.vgg16_bn() #bn stands for batch normalization.","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:34.936742Z","iopub.execute_input":"2023-05-01T15:28:34.937254Z","iopub.status.idle":"2023-05-01T15:28:37.215555Z","shell.execute_reply.started":"2023-05-01T15:28:34.937200Z","shell.execute_reply":"2023-05-01T15:28:37.214457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(vgg)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-01T15:28:37.217252Z","iopub.execute_input":"2023-05-01T15:28:37.217626Z","iopub.status.idle":"2023-05-01T15:28:37.224255Z","shell.execute_reply.started":"2023-05-01T15:28:37.217586Z","shell.execute_reply":"2023-05-01T15:28:37.222741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(vgg.features[0])","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.225762Z","iopub.execute_input":"2023-05-01T15:28:37.226067Z","iopub.status.idle":"2023-05-01T15:28:37.235756Z","shell.execute_reply.started":"2023-05-01T15:28:37.226040Z","shell.execute_reply":"2023-05-01T15:28:37.234597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(vgg.classifier[6])","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.237527Z","iopub.execute_input":"2023-05-01T15:28:37.238381Z","iopub.status.idle":"2023-05-01T15:28:37.246441Z","shell.execute_reply.started":"2023-05-01T15:28:37.238345Z","shell.execute_reply":"2023-05-01T15:28:37.245310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# last layer has 1000 classes, but for CIFAR10 has only 10 label outputs, changing the same with below code. \n\nfinal_in_features = vgg.classifier[6].in_features\nmod_classifier = list(vgg.classifier.children())[:-1] # keeping all the layers expect the last one\nmod_classifier.extend([nn.Linear(final_in_features, num_classes)]) # extending the last layer with the requiered number of class. \nprint(mod_classifier)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.248064Z","iopub.execute_input":"2023-05-01T15:28:37.248430Z","iopub.status.idle":"2023-05-01T15:28:37.262154Z","shell.execute_reply.started":"2023-05-01T15:28:37.248392Z","shell.execute_reply":"2023-05-01T15:28:37.260981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg.classifier = nn.Sequential(*mod_classifier)\nprint(vgg)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-05-01T15:28:37.263427Z","iopub.execute_input":"2023-05-01T15:28:37.264505Z","iopub.status.idle":"2023-05-01T15:28:37.272639Z","shell.execute_reply.started":"2023-05-01T15:28:37.264465Z","shell.execute_reply":"2023-05-01T15:28:37.271632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train CIFAR10","metadata":{}},{"cell_type":"code","source":"batch_size = 16\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.273796Z","iopub.execute_input":"2023-05-01T15:28:37.274733Z","iopub.status.idle":"2023-05-01T15:28:37.282992Z","shell.execute_reply.started":"2023-05-01T15:28:37.274698Z","shell.execute_reply":"2023-05-01T15:28:37.281857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation(dataloader, model):\n    total, correct = 0, 0\n    for data in dataloader:\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = model(inputs)\n        _, pred = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (pred == labels).sum().item()\n    return 100 * correct / total","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.284262Z","iopub.execute_input":"2023-05-01T15:28:37.287026Z","iopub.status.idle":"2023-05-01T15:28:37.295243Z","shell.execute_reply.started":"2023-05-01T15:28:37.286998Z","shell.execute_reply":"2023-05-01T15:28:37.294250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = vgg.to(device)\nloss_fn = nn.CrossEntropyLoss()\nopt = optim.SGD(vgg.parameters(), lr=0.05)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:37.296998Z","iopub.execute_input":"2023-05-01T15:28:37.297371Z","iopub.status.idle":"2023-05-01T15:28:40.087264Z","shell.execute_reply.started":"2023-05-01T15:28:37.297335Z","shell.execute_reply":"2023-05-01T15:28:40.086217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nloss_epoch_arr = []\nmax_epochs = 1\nclip_value = 5\n\nn_iters = np.ceil(50000/batch_size)\n\nfor epoch in range(max_epochs):\n\n    for i, data in enumerate(trainloader, 0):\n\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        opt.zero_grad()\n\n        outputs = vgg(inputs)\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        torch.nn.utils.clip_grad_norm_(vgg.parameters(), clip_value)\n        opt.step()\n        \n        # below steps ensures that the memory usage in the GPU is optimised. \n        del inputs, labels, outputs\n        torch.cuda.empty_cache()\n        \n        if i % 100 == 0:\n            print('Iteration: %d/%d, Loss: %0.2f' % (i, n_iters, loss.item()))\n        \n    loss_epoch_arr.append(loss.item())\n        \n    print('Epoch: %d/%d, Test acc: %0.2f, Train acc: %0.2f' % (\n        epoch, max_epochs, \n        evaluation(testloader, vgg), evaluation(trainloader, vgg)))\n    \n# change max_epochs value to > 1 \n#plt.plot(loss_epoch_arr) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:28:40.088721Z","iopub.execute_input":"2023-05-01T15:28:40.089091Z","iopub.status.idle":"2023-05-01T15:44:07.226679Z","shell.execute_reply.started":"2023-05-01T15:28:40.089052Z","shell.execute_reply":"2023-05-01T15:44:07.225493Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Freeze layers of Convolutional Operations\nThe above vgg16 module is taking a long time to train. To reduce the training time use a pre-trained model (the weights of the models have been saved). The approach is to freeze some layers and train the required layer to reduce the training time. ","metadata":{}},{"cell_type":"code","source":"batch_size = 16\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:07.228263Z","iopub.execute_input":"2023-05-01T15:44:07.228832Z","iopub.status.idle":"2023-05-01T15:44:07.235985Z","shell.execute_reply.started":"2023-05-01T15:44:07.228790Z","shell.execute_reply":"2023-05-01T15:44:07.233848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# downloading the pre-trained model. Please note VGGnet is trained for ImageNet dataset that had 1000 labels.\n\nvgg = models.vgg16_bn(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:07.237571Z","iopub.execute_input":"2023-05-01T15:44:07.238316Z","iopub.status.idle":"2023-05-01T15:44:17.105707Z","shell.execute_reply.started":"2023-05-01T15:44:07.238279Z","shell.execute_reply":"2023-05-01T15:44:17.104652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preventing the weights updates via back propegation.\n\nfor param in vgg.parameters():\n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:17.107515Z","iopub.execute_input":"2023-05-01T15:44:17.107947Z","iopub.status.idle":"2023-05-01T15:44:17.115303Z","shell.execute_reply.started":"2023-05-01T15:44:17.107888Z","shell.execute_reply":"2023-05-01T15:44:17.112990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adding the final layes as a linear layer with num of classes.\n# only this layer will have requires_grad as true, only weights of this layer get updated.\n\nfinal_in_features = vgg.classifier[6].in_features\nvgg.classifier[6] = nn.Linear(final_in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:17.117972Z","iopub.execute_input":"2023-05-01T15:44:17.118320Z","iopub.status.idle":"2023-05-01T15:44:17.194517Z","shell.execute_reply.started":"2023-05-01T15:44:17.118284Z","shell.execute_reply":"2023-05-01T15:44:17.193429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in vgg.parameters():\n    if param.requires_grad:\n        print(param.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:17.196017Z","iopub.execute_input":"2023-05-01T15:44:17.196472Z","iopub.status.idle":"2023-05-01T15:44:17.206582Z","shell.execute_reply.started":"2023-05-01T15:44:17.196433Z","shell.execute_reply":"2023-05-01T15:44:17.205398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg = vgg.to(device)\nloss_fn = nn.CrossEntropyLoss()\nopt = optim.SGD(vgg.parameters(), lr=0.05)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:17.207683Z","iopub.execute_input":"2023-05-01T15:44:17.207997Z","iopub.status.idle":"2023-05-01T15:44:17.362313Z","shell.execute_reply.started":"2023-05-01T15:44:17.207970Z","shell.execute_reply":"2023-05-01T15:44:17.361123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\nloss_epoch_arr = []\nmax_epochs = 1\n\nn_iters = np.ceil(50000/batch_size)\n\nfor epoch in range(max_epochs):\n\n    for i, data in enumerate(trainloader, 0):\n\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        opt.zero_grad()\n\n        outputs = vgg(inputs)\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        opt.step()\n        \n        if i % 100 == 0:\n            print('Iteration: %d/%d, Loss: %0.2f' % (i, n_iters, loss.item()))\n            \n        del inputs, labels, outputs\n        torch.cuda.empty_cache()\n        \n    loss_epoch_arr.append(loss.item())\n        \n    print('Epoch: %d/%d, Test acc: %0.2f, Train acc: %0.2f' % (\n        epoch, max_epochs, \n        evaluation(testloader, vgg), evaluation(trainloader, vgg)))\n\n# change max_epochs value to > 1 \n#plt.plot(loss_epoch_arr) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:44:17.369186Z","iopub.execute_input":"2023-05-01T15:44:17.369503Z","iopub.status.idle":"2023-05-01T15:53:07.258784Z","shell.execute_reply.started":"2023-05-01T15:44:17.369474Z","shell.execute_reply":"2023-05-01T15:53:07.257554Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# With model copies (Checkpoint)","metadata":{}},{"cell_type":"code","source":"import copy","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:53:07.260703Z","iopub.execute_input":"2023-05-01T15:53:07.261534Z","iopub.status.idle":"2023-05-01T15:53:07.266473Z","shell.execute_reply.started":"2023-05-01T15:53:07.261490Z","shell.execute_reply":"2023-05-01T15:53:07.265412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_epoch_arr = []\nmax_epochs = 1\n\nmin_loss = 1000\n\nn_iters = np.ceil(50000/batch_size)\n\nfor epoch in range(max_epochs):\n\n    for i, data in enumerate(trainloader, 0):\n\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        opt.zero_grad()\n\n        outputs = vgg(inputs)\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        opt.step()\n        \n        if min_loss > loss.item():\n            min_loss = loss.item()\n            best_model = copy.deepcopy(vgg.state_dict())\n            print('Min loss %0.2f' % min_loss)\n        \n        if i % 100 == 0:\n            print('Iteration: %d/%d, Loss: %0.2f' % (i, n_iters, loss.item()))\n            \n        del inputs, labels, outputs\n        torch.cuda.empty_cache()\n        \n    loss_epoch_arr.append(loss.item())\n\n# change max_epochs value to > 1 \n#plt.plot(loss_epoch_arr) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:53:07.268182Z","iopub.execute_input":"2023-05-01T15:53:07.268925Z","iopub.status.idle":"2023-05-01T15:57:41.447882Z","shell.execute_reply.started":"2023-05-01T15:53:07.268872Z","shell.execute_reply":"2023-05-01T15:57:41.446846Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vgg.load_state_dict(best_model)\nprint(evaluation(trainloader, vgg), evaluation(testloader, vgg))","metadata":{"execution":{"iopub.status.busy":"2023-05-01T15:57:41.449308Z","iopub.execute_input":"2023-05-01T15:57:41.450113Z","iopub.status.idle":"2023-05-01T16:01:56.565690Z","shell.execute_reply.started":"2023-05-01T15:57:41.450073Z","shell.execute_reply":"2023-05-01T16:01:56.564428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ResNet Model\nhttps://pytorch.org/vision/stable/models/resnet.html","metadata":{}},{"cell_type":"code","source":"resnet = models.resnet18(pretrained=True)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:56.567276Z","iopub.execute_input":"2023-05-01T16:01:56.567870Z","iopub.status.idle":"2023-05-01T16:01:57.267860Z","shell.execute_reply.started":"2023-05-01T16:01:56.567826Z","shell.execute_reply":"2023-05-01T16:01:57.266875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(resnet)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.269446Z","iopub.execute_input":"2023-05-01T16:01:57.270193Z","iopub.status.idle":"2023-05-01T16:01:57.277547Z","shell.execute_reply.started":"2023-05-01T16:01:57.270152Z","shell.execute_reply":"2023-05-01T16:01:57.276545Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in resnet.parameters():\n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.279588Z","iopub.execute_input":"2023-05-01T16:01:57.281373Z","iopub.status.idle":"2023-05-01T16:01:57.290424Z","shell.execute_reply.started":"2023-05-01T16:01:57.281335Z","shell.execute_reply":"2023-05-01T16:01:57.289481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_features = resnet.fc.in_features\nresnet.fc = nn.Linear(in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.292357Z","iopub.execute_input":"2023-05-01T16:01:57.293052Z","iopub.status.idle":"2023-05-01T16:01:57.304052Z","shell.execute_reply.started":"2023-05-01T16:01:57.293017Z","shell.execute_reply":"2023-05-01T16:01:57.303142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in resnet.parameters():\n    if param.requires_grad:\n        print(param.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.305170Z","iopub.execute_input":"2023-05-01T16:01:57.305537Z","iopub.status.idle":"2023-05-01T16:01:57.314979Z","shell.execute_reply.started":"2023-05-01T16:01:57.305503Z","shell.execute_reply":"2023-05-01T16:01:57.314053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet = resnet.to(device)\nloss_fn = nn.CrossEntropyLoss()\nopt = optim.SGD(resnet.parameters(), lr=0.01)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.316375Z","iopub.execute_input":"2023-05-01T16:01:57.317481Z","iopub.status.idle":"2023-05-01T16:01:57.352557Z","shell.execute_reply.started":"2023-05-01T16:01:57.317443Z","shell.execute_reply":"2023-05-01T16:01:57.351684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_epoch_arr = []\nmax_epochs = 1\n\nmin_loss = 1000\n\nn_iters = np.ceil(50000/batch_size)\n\nfor epoch in range(max_epochs):\n\n    for i, data in enumerate(trainloader, 0):\n\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        opt.zero_grad()\n\n        outputs = resnet(inputs)\n        loss = loss_fn(outputs, labels)\n        loss.backward()\n        opt.step()\n        \n        if min_loss > loss.item():\n            min_loss = loss.item()\n            best_model = copy.deepcopy(resnet.state_dict())\n            print('Min loss %0.2f' % min_loss)\n        \n        if i % 100 == 0:\n            print('Iteration: %d/%d, Loss: %0.2f' % (i, n_iters, loss.item()))\n            \n        del inputs, labels, outputs\n        torch.cuda.empty_cache()\n        \n    loss_epoch_arr.append(loss.item())\n        \n    print('Epoch: %d/%d, Test acc: %0.2f, Train acc: %0.2f' % (\n        epoch, max_epochs, \n        evaluation(testloader, resnet), evaluation(trainloader, resnet)))\n    \n    \n# change max_epochs value to > 1 \n#plt.plot(loss_epoch_arr) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:01:57.356419Z","iopub.execute_input":"2023-05-01T16:01:57.359062Z","iopub.status.idle":"2023-05-01T16:05:46.496196Z","shell.execute_reply.started":"2023-05-01T16:01:57.359026Z","shell.execute_reply":"2023-05-01T16:05:46.494948Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet.load_state_dict(best_model)\nprint(evaluation(trainloader, resnet), evaluation(testloader, resnet))","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:05:46.497694Z","iopub.execute_input":"2023-05-01T16:05:46.499198Z","iopub.status.idle":"2023-05-01T16:07:50.743056Z","shell.execute_reply.started":"2023-05-01T16:05:46.499155Z","shell.execute_reply":"2023-05-01T16:07:50.741907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inception Model\nhttps://pytorch.org/vision/stable/models/inception.html","metadata":{}},{"cell_type":"code","source":"inception = models.inception_v3(pretrained=True)\nprint(inception)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:50.744619Z","iopub.execute_input":"2023-05-01T16:07:50.745278Z","iopub.status.idle":"2023-05-01T16:07:52.925167Z","shell.execute_reply.started":"2023-05-01T16:07:50.745237Z","shell.execute_reply":"2023-05-01T16:07:52.924047Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in inception.parameters():\n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.926955Z","iopub.execute_input":"2023-05-01T16:07:52.927696Z","iopub.status.idle":"2023-05-01T16:07:52.934279Z","shell.execute_reply.started":"2023-05-01T16:07:52.927654Z","shell.execute_reply":"2023-05-01T16:07:52.933296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aux_in_features = inception.AuxLogits.fc.in_features\ninception.AuxLogits.fc = nn.Linear(aux_in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.935956Z","iopub.execute_input":"2023-05-01T16:07:52.936376Z","iopub.status.idle":"2023-05-01T16:07:52.945763Z","shell.execute_reply.started":"2023-05-01T16:07:52.936337Z","shell.execute_reply":"2023-05-01T16:07:52.944742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in inception.parameters():\n    if param.requires_grad:\n        print(param.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.948527Z","iopub.execute_input":"2023-05-01T16:07:52.948821Z","iopub.status.idle":"2023-05-01T16:07:52.958304Z","shell.execute_reply.started":"2023-05-01T16:07:52.948794Z","shell.execute_reply":"2023-05-01T16:07:52.957237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"in_features = inception.fc.in_features\ninception.fc = nn.Linear(in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.959631Z","iopub.execute_input":"2023-05-01T16:07:52.960401Z","iopub.status.idle":"2023-05-01T16:07:52.969186Z","shell.execute_reply.started":"2023-05-01T16:07:52.960365Z","shell.execute_reply":"2023-05-01T16:07:52.968132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for param in inception.parameters():\n    if param.requires_grad:\n        print(param.shape)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.971735Z","iopub.execute_input":"2023-05-01T16:07:52.972673Z","iopub.status.idle":"2023-05-01T16:07:52.981334Z","shell.execute_reply.started":"2023-05-01T16:07:52.972636Z","shell.execute_reply":"2023-05-01T16:07:52.980224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([\n    transforms.RandomResizedCrop(299), \n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])\n\ntransform_test = transforms.Compose([\n    transforms.RandomResizedCrop(299), \n    transforms.ToTensor(),\n    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),\n    ])","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.982622Z","iopub.execute_input":"2023-05-01T16:07:52.983138Z","iopub.status.idle":"2023-05-01T16:07:52.994252Z","shell.execute_reply.started":"2023-05-01T16:07:52.983103Z","shell.execute_reply":"2023-05-01T16:07:52.993195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainset = torchvision.datasets.CIFAR10(root='./data', train=True, \n                                        download=True, \n                                        transform=transform_train)\ntestset = torchvision.datasets.CIFAR10(root='./data', train=False, \n                                        download=True, \n                                        transform=transform_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:52.995476Z","iopub.execute_input":"2023-05-01T16:07:52.996126Z","iopub.status.idle":"2023-05-01T16:07:54.649604Z","shell.execute_reply.started":"2023-05-01T16:07:52.996087Z","shell.execute_reply":"2023-05-01T16:07:54.648565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=16\n\ntrainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)\ntestloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:54.651173Z","iopub.execute_input":"2023-05-01T16:07:54.651537Z","iopub.status.idle":"2023-05-01T16:07:54.657465Z","shell.execute_reply.started":"2023-05-01T16:07:54.651496Z","shell.execute_reply":"2023-05-01T16:07:54.656096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inception = inception.to(device)\nloss_fn = nn.CrossEntropyLoss()\nopt = optim.SGD(inception.parameters(), lr=0.01)","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:54.659072Z","iopub.execute_input":"2023-05-01T16:07:54.659752Z","iopub.status.idle":"2023-05-01T16:07:54.716022Z","shell.execute_reply.started":"2023-05-01T16:07:54.659716Z","shell.execute_reply":"2023-05-01T16:07:54.715088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def evaluation_inception(dataloader, model):\n    total, correct = 0, 0\n    for data in dataloader:\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs, aux_outputs = model(inputs)\n        _, pred = torch.max(outputs.data, 1)\n        total += labels.size(0)\n        correct += (pred == labels).sum().item()\n    return 100 * correct / total","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:54.717445Z","iopub.execute_input":"2023-05-01T16:07:54.717805Z","iopub.status.idle":"2023-05-01T16:07:54.724839Z","shell.execute_reply.started":"2023-05-01T16:07:54.717767Z","shell.execute_reply":"2023-05-01T16:07:54.723703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_epoch_arr = []\nmax_epochs = 1\n\nmin_loss = 1000\n\nn_iters = np.ceil(50000/batch_size)\n\nfor epoch in range(max_epochs):\n\n    for i, data in enumerate(trainloader, 0):\n\n        inputs, labels = data\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        opt.zero_grad()\n\n        outputs, aux_outputs = inception(inputs)\n        loss = loss_fn(outputs, labels) + 0.3 * loss_fn(aux_outputs, labels)\n        loss.backward()\n        opt.step()\n        \n        if min_loss > loss.item():\n            min_loss = loss.item()\n            best_model = copy.deepcopy(inception.state_dict())\n            print('Min loss %0.2f' % min_loss)\n        \n        if i % 100 == 0:\n            print('Iteration: %d/%d, Loss: %0.2f' % (i, n_iters, loss.item()))\n            \n        del inputs, labels, outputs\n        torch.cuda.empty_cache()\n        \n    loss_epoch_arr.append(loss.item())\n        \n    print('Epoch: %d/%d, Test acc: %0.2f, Train acc: %0.2f' % (\n        epoch, max_epochs, \n        evaluation_inception(testloader, inception), \n        evaluation_inception(trainloader, inception)))\n    \n    \n# change max_epochs value to > 1 \n#plt.plot(loss_epoch_arr) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:07:54.726483Z","iopub.execute_input":"2023-05-01T16:07:54.726836Z","iopub.status.idle":"2023-05-01T16:16:55.769853Z","shell.execute_reply.started":"2023-05-01T16:07:54.726799Z","shell.execute_reply":"2023-05-01T16:16:55.768620Z"},"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inception.load_state_dict(best_model)\nprint(evaluation_inception(trainloader, inception), evaluation_inception(testloader, inception))","metadata":{"execution":{"iopub.status.busy":"2023-05-01T16:16:55.771625Z","iopub.execute_input":"2023-05-01T16:16:55.772036Z","iopub.status.idle":"2023-05-01T16:21:42.506686Z","shell.execute_reply.started":"2023-05-01T16:16:55.771994Z","shell.execute_reply":"2023-05-01T16:21:42.505468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# END ","metadata":{}}]}