{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<a id=\"toc_section\"></a>\n## Contents of this notebook\n\n[**Raghav Rastogi**](https://www.kaggle.com/raghavrastogi75) \n\n\n* [Importing the libraries](#1)\n* [Loading and transforming the images](#2)\n* [Splitting the dataset](#3)\n* [Taking a look at the normalised images](#4)\n* [Creating Custom CNN network](#5)\n* [Transfer learning - VGG16](#6)\n* [Training Custom model](#7)\n* [Training VGG16 model](#8)","metadata":{}},{"cell_type":"code","source":"# Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        os.path.join(dirname, filename)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-03-24T23:18:33.57875Z","iopub.execute_input":"2022-03-24T23:18:33.579315Z","iopub.status.idle":"2022-03-24T23:21:50.934411Z","shell.execute_reply.started":"2022-03-24T23:18:33.579276Z","shell.execute_reply":"2022-03-24T23:21:50.933638Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Importing the libraries <span id=\"1\"></span>","metadata":{}},{"cell_type":"code","source":"\nimport torchvision\nimport torch\nimport numpy as np\nfrom torch import nn\nfrom torchvision import transforms, models, datasets\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport time\nimport pandas as pd\nimport torchvision.datasets as datasets\nfrom IPython import display\nimport matplotlib.pyplot as plt\nimport torchvision.transforms\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:21:50.935964Z","iopub.execute_input":"2022-03-24T23:21:50.936213Z","iopub.status.idle":"2022-03-24T23:21:52.550665Z","shell.execute_reply.started":"2022-03-24T23:21:50.936179Z","shell.execute_reply":"2022-03-24T23:21:52.549906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> #### Setting up the data folder","metadata":{}},{"cell_type":"code","source":"data_dir = \"../input/cats-vs-dogs-1/PetImages\"","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:21:52.552726Z","iopub.execute_input":"2022-03-24T23:21:52.55293Z","iopub.status.idle":"2022-03-24T23:21:52.559056Z","shell.execute_reply.started":"2022-03-24T23:21:52.552906Z","shell.execute_reply":"2022-03-24T23:21:52.558287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading and transforming the images <span id=\"2\"></span>","metadata":{}},{"cell_type":"markdown","source":"> #### The images are augmented and normalised here. We use \"ImageFolder\" where we have the directory in the specific format of images divided by each class as folders and path being defined as the main folder.","metadata":{}},{"cell_type":"code","source":"data_transforms = transforms.Compose([transforms.Resize([64,64]),\n                                      transforms.ToTensor(),\n                                      transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))\n                                     ])\n\ntraining_data = datasets.ImageFolder(data_dir,transform = data_transforms)\n\n","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:21:52.561233Z","iopub.execute_input":"2022-03-24T23:21:52.561793Z","iopub.status.idle":"2022-03-24T23:21:57.193329Z","shell.execute_reply.started":"2022-03-24T23:21:52.561756Z","shell.execute_reply":"2022-03-24T23:21:57.192594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting the dataset  <span id=\"3\"></span>","metadata":{}},{"cell_type":"markdown","source":"> #### I found out there are a lot are ways we can get the image data where the data is present as test and train directories, splitted by classes, or just a whole dump. This way below seems to be appropiate if get the format where the images are split by class as folders.\n\n> #### Here we simply split the training data to training, test and validation and get the loaders. Loaders is the 4 dimensional tensor which has (batch_size, channel, width, height) format.","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import SubsetRandomSampler\n\nvalid_size = 0.20\ntest_size = 0.10\n# Split data into train and validation set\nnum_train = len(training_data)\nindices = list(range(num_train))\nnp.random.shuffle(indices)\n#print(indices)\nsplit1 = int(np.floor(valid_size * num_train))\nsplit2 = int(np.floor(test_size*num_train)) + split1\n#print(split1,split2)\nvalid_idx, test_idx, train_idx = indices[:split1], indices[split1:split2],indices[split2:]\n\ntraining_data1 = SubsetRandomSampler(train_idx)\nvalid_data1 = SubsetRandomSampler(valid_idx)\ntest_data1 = SubsetRandomSampler(test_idx)\n\ntraining_loader = DataLoader(training_data, sampler = training_data1,batch_size = 25)\nvalid_loader = DataLoader(training_data, sampler = valid_data1, batch_size = 25)\ntest_loader = DataLoader(training_data, sampler = test_data1,batch_size = 25)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:21:57.195514Z","iopub.execute_input":"2022-03-24T23:21:57.196025Z","iopub.status.idle":"2022-03-24T23:21:57.206567Z","shell.execute_reply.started":"2022-03-24T23:21:57.195985Z","shell.execute_reply":"2022-03-24T23:21:57.205712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Taking a look at the normalised images <span id=\"4\"></span>","metadata":{}},{"cell_type":"code","source":"g = 1\nrows = 4\ncols = 4\nfigure = plt.figure(figsize=(16, 16))\nfor i,j in training_loader:    \n    figure.add_subplot(rows, cols, g)\n    for img in i:        \n        img = img.swapaxes(0, 1)\n        img = img.swapaxes(1, 2)\n        plt.imshow(img, cmap=\"gray\", interpolation='nearest', aspect='auto')        \n        break\n    g += 1    \n    if g > 16:\n        break\nplt.show()       ","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:21:57.207905Z","iopub.execute_input":"2022-03-24T23:21:57.208215Z","iopub.status.idle":"2022-03-24T23:22:02.353404Z","shell.execute_reply.started":"2022-03-24T23:21:57.208156Z","shell.execute_reply":"2022-03-24T23:22:02.352643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Custom CNN network <span id=\"5\"></span>","metadata":{}},{"cell_type":"markdown","source":"> ### Here are the key points when making a custom CNN model\n\n> * The formula O = ((I - f + 2p)/s) + 1 is used to keep track of the output image size where O is the output image size, f is the kernal size, p is the padding, s is the stride.\n\n> * Paramenters are calculated by ((NxNxC + 1)K) where N is the kernal size , C is the input channel and K is the output channels\n\n> * In the model below the kernal_size of 3 and padding 1 means that the outout image size is equal to the input image size. Howvever, the MaxPool2d, which has the kernal_size of 2 and stride of 2 halves the height and width of the image.\n\n> * Linear layer nodes are calculated by the product of input image size (NxN) and the out oput channels. Here it's 4x4x256\n\n> * Parameters of the Linear layer is calculates by (N1+1)N2 where N1 is the number of nodes in the previous layer and N2 is the number of nodes in the current layer\n","metadata":{}},{"cell_type":"code","source":"device =  torch.device('cuda' if torch.cuda.is_available else 'cpu')\n\nclass CNNModel(nn.Module):\n    def __init__(self, num_classes):\n\n        super(CNNModel, self).__init__()\n        self.features = nn.Sequential(\n        nn.Conv2d(3, 64, kernel_size=3, padding=1), #input image 64x64 - Parameters = (3x3x3 + 1)*64\n        nn.ReLU(),\n        nn.MaxPool2d(kernel_size = 2, stride = 2),\n        nn.Conv2d(64,128, kernel_size = 3,padding = 1), #input image 32x32 - Parameters = (3x3x64 + 1)*128\n        nn.ReLU(),\n        nn.MaxPool2d(kernel_size = 2, stride = 2),\n        nn.Conv2d(128, 256, kernel_size=3, padding=1), #input image 16x16 - Parameters = (3x3x128 + 1)*256\n        nn.ReLU(),\n        nn.MaxPool2d(kernel_size=2, stride=2),\n        nn.Conv2d(256, 256, kernel_size=3, padding=1), #input image 8x8 - Parameters = (3x3x256 + 1)*256\n        nn.ReLU(),\n        nn.MaxPool2d(kernel_size=2, stride=2) #output image 4x4\n        )\n        \n        self.classifier = nn.Sequential(\n        nn.Dropout(),\n        nn.Linear(4*4*256,1024), #Parameters - (4x4x256 + 1)(1024)\n        nn.ReLU(),\n        nn.Dropout(),\n        nn.Linear(1024, num_classes) #Parameters - (1024 + 1)(2)\n        )\n\n    def forward(self, x):\n        x = self.features(x)\n        x = torch.flatten(x, 1)\n        x = self.classifier(x)\n        return x\n    \n    \nmodel = CNNModel(num_classes=2)\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(model.parameters(), lr= 0.001)\nmodel.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:22:02.354477Z","iopub.execute_input":"2022-03-24T23:22:02.354744Z","iopub.status.idle":"2022-03-24T23:22:05.329941Z","shell.execute_reply.started":"2022-03-24T23:22:02.354706Z","shell.execute_reply":"2022-03-24T23:22:05.329258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"# Transfer learning - VGG16 <span id=\"6\"></span>","metadata":{}},{"cell_type":"markdown","source":"> * #### This is pretty straight forward. We freeze the CNN part of the network and train the linear network completely with the images.","metadata":{}},{"cell_type":"code","source":"model1 = models.vgg16(pretrained=True)\nfor param in model1.parameters():\n    param.requires_grad = False\nmodel1.avgpool = nn.AdaptiveAvgPool2d(output_size=(1,1))\nmodel1.classifier = nn.Sequential(nn.Flatten(),\nnn.Linear(512, 128),\nnn.ReLU(),\nnn.Dropout(0.2),\nnn.Linear(128, 2),\nnn.Sigmoid())\n\ncriterion1 = nn.BCELoss()\noptimizer1 = torch.optim.Adam(model1.parameters(), lr= 1e-3)\n\nmodel1.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:22:05.331092Z","iopub.execute_input":"2022-03-24T23:22:05.331331Z","iopub.status.idle":"2022-03-24T23:22:18.551258Z","shell.execute_reply.started":"2022-03-24T23:22:05.331298Z","shell.execute_reply":"2022-03-24T23:22:18.550409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training the model","metadata":{}},{"cell_type":"markdown","source":"> * #### I found the code below pretty standard and being used a lot of times and we simply call it to train any tyoe of model.","metadata":{}},{"cell_type":"code","source":"import time\n\ndef train(model, optimizer, loss_fn, train_dl, val_dl, epochs=100, device='cpu'):\n\n    print('train() called: model=%s, opt=%s(lr=%f), epochs=%d, device=%s\\n' % \\\n          (type(model).__name__, type(optimizer).__name__,\n           optimizer.param_groups[0]['lr'], epochs, device))\n\n    history = {} # Collects per-epoch loss and acc like Keras' fit().\n    history['loss'] = []\n    history['val_loss'] = []\n    history['acc'] = []\n    history['val_acc'] = []\n\n    start_time_sec = time.time()\n\n    for epoch in range(1, epochs+1):\n\n        # --- TRAIN AND EVALUATE ON TRAINING SET -----------------------------\n        model.train()\n        train_loss         = 0.0\n        num_train_correct  = 0\n        num_train_examples = 0\n\n        for batch in train_dl:\n\n            optimizer.zero_grad()\n\n            x    = batch[0].to(device)\n            y    = batch[1].to(device)\n            yhat = model(x)\n            loss = loss_fn(yhat, y)\n\n            loss.backward()\n            optimizer.step()\n\n            train_loss         += loss.data.item() * x.size(0)\n            num_train_correct  += (torch.max(yhat, 1)[1] == y).sum().item()\n            num_train_examples += x.shape[0]\n\n        train_acc   = num_train_correct / num_train_examples\n        train_loss  = train_loss / len(train_dl.dataset)\n\n\n        # --- EVALUATE ON VALIDATION SET -------------------------------------\n        model.eval()\n        val_loss       = 0.0\n        num_val_correct  = 0\n        num_val_examples = 0\n\n        for batch in val_dl:\n\n            x    = batch[0].to(device)\n            y    = batch[1].to(device)\n            yhat = model(x)\n            loss = loss_fn(yhat, y)\n\n            val_loss         += loss.data.item() * x.size(0)\n            num_val_correct  += (torch.max(yhat, 1)[1] == y).sum().item()\n            num_val_examples += y.shape[0]\n\n        val_acc  = num_val_correct / num_val_examples\n        val_loss = val_loss / len(val_dl.dataset)\n\n\n        if epoch == 1 or epoch % 10 == 0:\n          print('Epoch %3d/%3d, train loss: %5.2f, train acc: %5.2f, val loss: %5.2f, val acc: %5.2f' % \\\n                (epoch, epochs, train_loss, train_acc, val_loss, val_acc))\n\n        history['loss'].append(train_loss)\n        history['val_loss'].append(val_loss)\n        history['acc'].append(train_acc)\n        history['val_acc'].append(val_acc)\n\n    # END OF TRAINING LOOP\n\n\n    end_time_sec       = time.time()\n    total_time_sec     = end_time_sec - start_time_sec\n    time_per_epoch_sec = total_time_sec / epochs\n    print()\n    print('Time total:     %5.2f sec' % (total_time_sec))\n    print('Time per epoch: %5.2f sec' % (time_per_epoch_sec))\n\n    return history","metadata":{"execution":{"iopub.status.busy":"2022-03-24T23:22:18.552966Z","iopub.execute_input":"2022-03-24T23:22:18.553266Z","iopub.status.idle":"2022-03-24T23:22:18.570126Z","shell.execute_reply.started":"2022-03-24T23:22:18.553226Z","shell.execute_reply":"2022-03-24T23:22:18.56924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training Custom model <span id=\"7\"></span>","metadata":{}},{"cell_type":"code","source":"history = train(model, optimizer,criterion, training_loader,valid_loader, epochs=10, device=device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\nacc = history['acc']\nval_acc = history['val_acc']\nloss = history['loss']\nval_loss = history['val_loss']\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We get the validation accuracy to be around 0.82 for the custom CNN model","metadata":{}},{"cell_type":"markdown","source":"# Training VGG16 model <span id=\"8\"></span>","metadata":{}},{"cell_type":"code","source":"history1 = train(model1, optimizer1,criterion, training_loader,valid_loader, epochs=10, device=device)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history1['acc']\nval_acc = history1['val_acc']\nloss = history1['loss']\nval_loss = history1['val_loss']\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'b', label='Training acc')\nplt.plot(epochs, val_acc, 'r', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, 'b', label='Training loss')\nplt.plot(epochs, val_loss, 'r', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We get the validation accuracy to be around 0.9 for the pre-trained model which makes sense since the pretrained model should show better results","metadata":{}}]}