{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Let's break the notebook into separate steps. Feel free to navigate the notebook and comment if you have any suggestions.\n\nStep 0: Import Datasets \\\nStep 1: Detect Dogs\\\nStep 2: Create a CNN to Classify Dog Breeds (from Scratch)\\\nStep 3: Create a CNN to Classify Dog Breeds (using Transfer Learning)\\\nStep 4: Test","metadata":{}},{"cell_type":"markdown","source":"# Initializations\nAt first we need to import the libraries. It is considered as standard imports.\n","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader\nfrom torchvision import datasets, transforms, models\nfrom torchvision.utils import make_grid\n\nfrom PIL import Image\nfrom IPython.display import display\nimport cv2\nfrom PIL import ImageFile\nimport torchvision.transforms as transforms\nImageFile.LOAD_TRUNCATED_IMAGES = True\n\nimport glob\nimport os\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-06T09:45:38.299412Z","iopub.execute_input":"2021-11-06T09:45:38.299744Z","iopub.status.idle":"2021-11-06T09:45:40.144137Z","shell.execute_reply.started":"2021-11-06T09:45:38.299660Z","shell.execute_reply":"2021-11-06T09:45:40.143333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will be using this function mostly everywhere to run our experiments deterministically. Random functions of Numpy and Pandas will behave deterministically after this. To learn more about Deterministic Neural Networks please check out [this notebook](https://www.kaggle.com/bminixhofer/deterministic-neural-networks-using-pytorch)","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed=1234):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.146031Z","iopub.execute_input":"2021-11-06T09:45:40.146289Z","iopub.status.idle":"2021-11-06T09:45:40.156149Z","shell.execute_reply.started":"2021-11-06T09:45:40.146253Z","shell.execute_reply":"2021-11-06T09:45:40.155312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Have you wondered about why they use 42? Do you want to know about the reason behind 42? Look [Here ](https://en.wikipedia.org/wiki/Phrases_from_The_Hitchhiker%27s_Guide_to_the_Galaxy#Answer_to_the_Ultimate_Question_of_Life.2C_the_Universe_and_Everything_.2842.29):p\n\n","metadata":{}},{"cell_type":"code","source":"#Read the dataset \nPATH = '../input/dog-breed-identification/'\nlabels = pd.read_csv(PATH+'labels.csv')\nlabelnames = pd.read_csv(PATH + 'sample_submission.csv').keys()[1:]\nprint(\"Train folder has \", len(os.listdir(PATH+'train')),'images which matches with label\\'s', len(labels),'images')","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.158799Z","iopub.execute_input":"2021-11-06T09:45:40.159917Z","iopub.status.idle":"2021-11-06T09:45:40.878463Z","shell.execute_reply.started":"2021-11-06T09:45:40.159886Z","shell.execute_reply":"2021-11-06T09:45:40.877545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"codes = range(len(labelnames))\nbreed_to_code = dict(zip(labelnames, codes))\ncode_to_breed = dict(zip(codes, labelnames))\nlabels['target'] =  [breed_to_code[x] for x in labels.breed]\nlabels['rank'] = labels.groupby('breed').rank()['id']\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\n\ntrain = labels_pivot.sample(frac=0.85)\nvalid = labels_pivot[~labels_pivot['id'].isin(train['id'])]\nprint(train.shape, valid.shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.880759Z","iopub.execute_input":"2021-11-06T09:45:40.881068Z","iopub.status.idle":"2021-11-06T09:45:40.966206Z","shell.execute_reply.started":"2021-11-06T09:45:40.881029Z","shell.execute_reply":"2021-11-06T09:45:40.965491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.967680Z","iopub.execute_input":"2021-11-06T09:45:40.967980Z","iopub.status.idle":"2021-11-06T09:45:40.985051Z","shell.execute_reply.started":"2021-11-06T09:45:40.967944Z","shell.execute_reply":"2021-11-06T09:45:40.984237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using the code cell below to write three separate [data loaders](http://pytorch.org/docs/stable/data.html#torch.utils.data.DataLoader) for the training, validation, and test datasets of dog images (located at `dog_images/train`, `dog_images/valid`, and `dog_images/test`, respectively).  You may find [this documentation on custom datasets](https://pytorch.org/vision/stable/datasets.html) to be a useful resource.  If you are interested in augmenting your training and/or validation data, check out the wide variety of [transforms](https://pytorch.org/vision/stable/transforms.html)!","metadata":{}},{"cell_type":"code","source":" # Image transformations\nimg_transform = {\n    'valid':transforms.Compose([\n        transforms.Resize(size = 256),\n        transforms.CenterCrop(size = 224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ]),\n    'train':transforms.Compose([\n        transforms.RandomResizedCrop(size = 256),\n        transforms.RandomRotation(degrees = 30),\n        transforms.ColorJitter(),\n        transforms.RandomHorizontalFlip(),\n        transforms.CenterCrop(size=224),  \n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])  \n    ]),\n    'test':transforms.Compose([\n        transforms.Resize(size = 256),\n        transforms.CenterCrop(size = 224),\n        transforms.ToTensor(),\n        transforms.Normalize([0.485, 0.456, 0.406],\n                             [0.229, 0.224, 0.225])\n    ]),\n}","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.986574Z","iopub.execute_input":"2021-11-06T09:45:40.986869Z","iopub.status.idle":"2021-11-06T09:45:40.996946Z","shell.execute_reply.started":"2021-11-06T09:45:40.986811Z","shell.execute_reply":"2021-11-06T09:45:40.995925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I resized the pictures to 256256 and then cropped the image into 224224 randomly to avoid squashed images and normalized it using Imagenet's mean and standard deviation after converting to tensor. for train, test and valid set.\n\nFor training images, I used data augmentation which includes random rotation of 30 degrees and horizontal flip.","metadata":{}},{"cell_type":"code","source":"class DogBreedDataset(torch.utils.data.Dataset):\n    'Characterizes a dataset for PyTorch'\n    def __init__(self, img_dir, label, transform):\n        'Initialization'\n        self.img_dir = img_dir\n        self.transform = transform\n        self.label = label\n\n    def __len__(self):\n        'Denotes the total number of samples'\n        return len(self.label)\n\n    def __getitem__(self, index):\n        if self.label is not None:\n            img_name = '{}.jpg'.format(self.label.iloc[index, 0])\n            fullname = self.img_dir + img_name\n            image = Image.open(fullname)\n            label = self.label.iloc[index, 1:].astype('float').to_numpy()\n            label = np.argmax(label)\n            if self.transform:\n                image = self.transform(image)\n            return [image, label]\n        ","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:40.998734Z","iopub.execute_input":"2021-11-06T09:45:40.999105Z","iopub.status.idle":"2021-11-06T09:45:41.009872Z","shell.execute_reply.started":"2021-11-06T09:45:40.999066Z","shell.execute_reply":"2021-11-06T09:45:41.009171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 12\nnum_workers = 4\ntrain_img = DogBreedDataset(PATH+'train/', train, transform = img_transform['train'])\nvalid_img = DogBreedDataset(PATH+'train/', valid, transform = img_transform['valid'])\n\n\ndataloaders={\n    'train':torch.utils.data.DataLoader(train_img, batch_size, num_workers = num_workers, shuffle=True),\n    'valid':torch.utils.data.DataLoader(valid_img, batch_size, num_workers = num_workers, shuffle=False)\n}\n","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:41.011272Z","iopub.execute_input":"2021-11-06T09:45:41.011573Z","iopub.status.idle":"2021-11-06T09:45:41.020273Z","shell.execute_reply.started":"2021-11-06T09:45:41.011522Z","shell.execute_reply":"2021-11-06T09:45:41.019345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use_cuda = torch.cuda.is_available()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:41.021581Z","iopub.execute_input":"2021-11-06T09:45:41.023106Z","iopub.status.idle":"2021-11-06T09:45:41.089829Z","shell.execute_reply.started":"2021-11-06T09:45:41.023070Z","shell.execute_reply":"2021-11-06T09:45:41.089020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def imshow(axis, inp):\n    \"\"\"Denormalize and show\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    axis.imshow(inp)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:41.092966Z","iopub.execute_input":"2021-11-06T09:45:41.093154Z","iopub.status.idle":"2021-11-06T09:45:41.098558Z","shell.execute_reply.started":"2021-11-06T09:45:41.093130Z","shell.execute_reply":"2021-11-06T09:45:41.097914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from mpl_toolkits.axes_grid1 import ImageGrid\nimg, label = next(iter(dataloaders['train']))\nprint(img.size(), label.size())\nfig = plt.figure(1, figsize=(16, 12))\ngrid = ImageGrid(fig, 111, nrows_ncols=(3, 4), axes_pad=0.05)    \nfor i in range(img.size()[0]):\n    ax = grid[i]\n    imshow(ax, img[i])","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:41.099810Z","iopub.execute_input":"2021-11-06T09:45:41.100564Z","iopub.status.idle":"2021-11-06T09:45:44.127117Z","shell.execute_reply.started":"2021-11-06T09:45:41.100529Z","shell.execute_reply":"2021-11-06T09:45:44.126444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN Model from scratch","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\n# define the CNN architecture\nclass Net(nn.Module):\n    ### TODO: choose an architecture, and complete the class\n    def __init__(self):\n        super(Net, self).__init__()\n        ## Define layers of a CNN\n    \n        # Convolution layers\n        self.conv1 = nn.Conv2d(3, 32, 3, padding = 1)\n        self.conv2 = nn.Conv2d(32, 64, 3, padding = 1)\n        self.conv3 = nn.Conv2d(64, 128, 3, padding = 1)\n        \n        # Max pooling layer (divides image size by 2)\n        self.pool = nn.MaxPool2d(2, 2)\n        \n        # Fully connected layers\n        self.fc1 = nn.Linear(128 * 28 * 28, 500)\n        self.fc2 = nn.Linear(500, 120)\n        \n        # Dropout\n        self.dropout = nn.Dropout(0.3)\n        \n        \n    def forward(self, x):\n        ## Define forward behavior\n        \n        # Sequence of convolutional and max pooling layers\n        x = self.pool(F.relu(self.conv1(x)))\n        x = self.pool(F.relu(self.conv2(x)))\n        x = self.pool(F.relu(self.conv3(x)))\n        # Flatten image input\n        x = x.view(-1, 128 * 28 * 28)\n        # Dropout layer\n        x = self.dropout(x)\n        # 1st hidden layer, with relu activation function\n        x = F.relu(self.fc1(x))\n        # Dropout layer\n        x = self.dropout(x)\n        # 2nd hidden layer\n        x = self.fc2(x)\n        return x\n\n# instantiate the CNN\nmodel_scratch = Net()\n\n# move tensors to GPU if CUDA is available\nif use_cuda:\n    model_scratch.cuda()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:44.128171Z","iopub.execute_input":"2021-11-06T09:45:44.128409Z","iopub.status.idle":"2021-11-06T09:45:47.476580Z","shell.execute_reply.started":"2021-11-06T09:45:44.128379Z","shell.execute_reply":"2021-11-06T09:45:47.475844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I used three convolutional layers with relu activations which are followed by maxpool layers. Also, used two fully connected layers. Between fully connected layers, dropout technique with probability = 0.25 is used to avoid the overfitting.","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:47.477877Z","iopub.execute_input":"2021-11-06T09:45:47.478117Z","iopub.status.idle":"2021-11-06T09:45:56.317459Z","shell.execute_reply.started":"2021-11-06T09:45:47.478084Z","shell.execute_reply":"2021-11-06T09:45:56.316623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#let's see the model\nfrom torchsummary import summary\nsummary(model_scratch, input_size=(3, 224, 224))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:45:56.320315Z","iopub.execute_input":"2021-11-06T09:45:56.320599Z","iopub.status.idle":"2021-11-06T09:46:01.970868Z","shell.execute_reply.started":"2021-11-06T09:45:56.320556Z","shell.execute_reply":"2021-11-06T09:46:01.970121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"the next code cell to specify a [loss function](http://pytorch.org/docs/stable/nn.html#loss-functions) and [optimizer](http://pytorch.org/docs/stable/optim.html).  ","metadata":{}},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(model_scratch.parameters(), lr=0.01, momentum = 0.9)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:46:01.972508Z","iopub.execute_input":"2021-11-06T09:46:01.972805Z","iopub.status.idle":"2021-11-06T09:46:01.979642Z","shell.execute_reply.started":"2021-11-06T09:46:01.972755Z","shell.execute_reply":"2021-11-06T09:46:01.978599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(n_epochs, loaders, model, optimizer, criterion, use_cuda, save_path):\n    \"\"\"returns trained model\"\"\"\n    # initialize tracker for minimum validation loss\n    valid_loss_min = np.Inf \n    \n    for epoch in range(1, n_epochs+1):\n        # initialize variables to monitor training and validation loss\n        train_loss = 0.0\n        valid_loss = 0.0\n        \n        ###################\n        # train the model #\n        ###################\n        model.train()\n        for batch_idx, (data, target) in enumerate(loaders['train']):\n            # move to GPU\n            if use_cuda:\n                data, target = data.cuda(), target.cuda()\n            ## find the loss and update the model parameters accordingly\n            ## record the average training loss, using something like\n            \n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n            loss.backward()\n            optimizer.step()\n            train_loss = train_loss + ((1 / (batch_idx + 1)) * (loss.data - train_loss))\n            \n            if batch_idx % 100 == 0:\n                print('Epoch: %d \\tBatch: %d \\tTraining Loss: %.6f' %(epoch, batch_idx + 1, train_loss))\n        ######################    \n        # validate the model #\n        ######################\n        model.eval()\n        for batch_idx, (data, target) in enumerate(loaders['valid']):\n            # move to GPU\n            if use_cuda:\n                data, target = data.cuda(), target.cuda()\n            ## update the average validation loss\n            output = model(data)\n            loss = criterion(output, target)\n            valid_loss = valid_loss + ((1 / (batch_idx + 1)) * (loss.data - valid_loss))\n            \n        # print training/validation statistics \n        print('Epoch: {} \\tTraining Loss: {:.4f} \\tValidation Loss: {:.4f}'.format(\n            epoch, \n            train_loss,\n            valid_loss\n            ))\n        \n        ## TODO: save the model if validation loss has decreased\n        if valid_loss < valid_loss_min:\n            torch.save(model.state_dict(), save_path)\n            print('BOOM! Validation loss decreased ({:.4f} --> {:.4f}).  Saving model...'.format(valid_loss_min,valid_loss))\n            valid_loss_min = valid_loss    \n\n    # return trained model\n    return model\n# train the model\nmodel_scratch = train(10, dataloaders, model_scratch, optimizer, \n                      criterion, use_cuda, 'model_scratch.pt')\n\n# load the model that got the best validation accuracy\nmodel_scratch.load_state_dict(torch.load('model_scratch.pt'))","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:46:01.981126Z","iopub.execute_input":"2021-11-06T09:46:01.981729Z","iopub.status.idle":"2021-11-06T09:56:14.974525Z","shell.execute_reply.started":"2021-11-06T09:46:01.981683Z","shell.execute_reply":"2021-11-06T09:56:14.973080Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"as you can see it has a pretty big loss value. Training a model from scratch and getting good loss can be hard with epochs like 10. So let's move to transfer learning models which are pretrained. ","metadata":{}},{"cell_type":"code","source":"resnet50 = models.resnet50(pretrained=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:56:14.976505Z","iopub.execute_input":"2021-11-06T09:56:14.976795Z","iopub.status.idle":"2021-11-06T09:56:21.273143Z","shell.execute_reply.started":"2021-11-06T09:56:14.976756Z","shell.execute_reply":"2021-11-06T09:56:21.272343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##  Specify model architecture \nmodel_transfer = models.resnet50(pretrained=True)\n\n# Freeze training for all \"features\" layers\nfor param in model_transfer.parameters():\n    param.requires_grad = False\n    \n# replace the last fully connected layer with a Linnear layer 133 output\nin_features = model_transfer.fc.in_features\nmodel_transfer.fc = nn.Linear(in_features, 120)\n\nif use_cuda:\n    model_transfer = model_transfer.cuda()","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:56:21.274583Z","iopub.execute_input":"2021-11-06T09:56:21.274844Z","iopub.status.idle":"2021-11-06T09:56:21.746426Z","shell.execute_reply.started":"2021-11-06T09:56:21.274799Z","shell.execute_reply":"2021-11-06T09:56:21.745678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion_transfer = nn.CrossEntropyLoss()\nmodel_transfer_grad_paramaters = filter(lambda p: p.requires_grad, model_transfer.parameters())\noptimizer_transfer = torch.optim.SGD(model_transfer_grad_paramaters, lr=0.01)","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:56:21.747808Z","iopub.execute_input":"2021-11-06T09:56:21.748085Z","iopub.status.idle":"2021-11-06T09:56:21.753679Z","shell.execute_reply.started":"2021-11-06T09:56:21.748051Z","shell.execute_reply":"2021-11-06T09:56:21.753022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 10\n# train the model\nmodel_transfer =  train(n_epochs, dataloaders, model_transfer, optimizer_transfer, criterion_transfer, use_cuda, 'model_transfer.pt')\n","metadata":{"execution":{"iopub.status.busy":"2021-11-06T09:56:21.755070Z","iopub.execute_input":"2021-11-06T09:56:21.755672Z","iopub.status.idle":"2021-11-06T10:07:32.808098Z","shell.execute_reply.started":"2021-11-06T09:56:21.755632Z","shell.execute_reply":"2021-11-06T10:07:32.807222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SubmissionAdam","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(PATH+'/sample_submission.csv')\noutput = pd.DataFrame(index=submission.index, columns=submission.keys() )\noutput['id'] = submission['id']\nsubmission['target'] =  [0] * len(submission)\n\n#will do this part later :3 ","metadata":{"execution":{"iopub.status.busy":"2021-11-06T10:07:32.809686Z","iopub.execute_input":"2021-11-06T10:07:32.810404Z","iopub.status.idle":"2021-11-06T10:07:33.146994Z","shell.execute_reply.started":"2021-11-06T10:07:32.810360Z","shell.execute_reply":"2021-11-06T10:07:33.146270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}