{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#\"\"\"\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n#\"\"\"\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2022-04-29T18:46:06.153033Z","iopub.execute_input":"2022-04-29T18:46:06.154004Z","iopub.status.idle":"2022-04-29T18:46:06.161036Z","shell.execute_reply.started":"2022-04-29T18:46:06.153956Z","shell.execute_reply":"2022-04-29T18:46:06.160061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\nimport time\nfrom PIL import Image\n\nfrom matplotlib import pyplot as plt\n%matplotlib inline\n\npd.set_option('display.max_rows', None)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.353655Z","iopub.execute_input":"2022-04-29T18:46:06.356408Z","iopub.status.idle":"2022-04-29T18:46:06.378354Z","shell.execute_reply.started":"2022-04-29T18:46:06.356341Z","shell.execute_reply":"2022-04-29T18:46:06.377145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ROOT_PATH = '/kaggle/input/dog-breed-identification'\nprint(os.listdir(ROOT_PATH))\nTrain_path = ROOT_PATH + '/train'\nTest_path = ROOT_PATH + '/test'\n\nTrain_files = os.listdir(Train_path)\nTest_files = os.listdir(Test_path)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.501234Z","iopub.execute_input":"2022-04-29T18:46:06.501875Z","iopub.status.idle":"2022-04-29T18:46:06.524230Z","shell.execute_reply.started":"2022-04-29T18:46:06.501823Z","shell.execute_reply":"2022-04-29T18:46:06.523181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_df = pd.read_csv(f'{ROOT_PATH}/labels.csv')\n\nle = LabelEncoder()\nlabels_df['label'] = le.fit_transform(labels_df.breed)\nprint('Unique breeds : ', len(labels_df.breed.unique()))\nlabels_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.559621Z","iopub.execute_input":"2022-04-29T18:46:06.560228Z","iopub.status.idle":"2022-04-29T18:46:06.599515Z","shell.execute_reply.started":"2022-04-29T18:46:06.560178Z","shell.execute_reply":"2022-04-29T18:46:06.598053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RunAll = False\n\nif RunAll:\n    min_x = 500\n    min_y = 500\n    for row in labels_df.iterrows():\n        file_id = row[1]['id']\n        image_path = Train_path + '/' + file_id + '.jpg'\n        image = Image.open(image_path)\n        image_data = np.array(image)\n        x, y, z = image_data.shape\n        if x < min_x:\n            min_x = x\n        if y < min_y:\n            min_y = y\n        if z != 3:\n            print(z)\n            print(image_data.shape)    \n\n    print(min_x, min_y)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.614814Z","iopub.execute_input":"2022-04-29T18:46:06.615097Z","iopub.status.idle":"2022-04-29T18:46:06.624688Z","shell.execute_reply.started":"2022-04-29T18:46:06.615066Z","shell.execute_reply":"2022-04-29T18:46:06.623092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DogDataset(Dataset):\n    def __init__(self, data, path):\n        self.data = data\n        self.path = path\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, idx):\n        file_id = self.data.loc[idx, 'id']\n        label = self.data.loc[idx, 'label']\n        \n        image_path = self.path + '/' + file_id + '.jpg'\n        image = Image.open(image_path)\n        process = transforms.Compose([\n                    transforms.Resize(size=(224, 224)),\n                    transforms.ToTensor()\n                    ])\n        \n        x = process(image)\n\n        return {\n            'image_data': x,\n            'labels': torch.tensor(label)\n        }","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.670891Z","iopub.execute_input":"2022-04-29T18:46:06.671656Z","iopub.status.idle":"2022-04-29T18:46:06.681120Z","shell.execute_reply.started":"2022-04-29T18:46:06.671604Z","shell.execute_reply":"2022-04-29T18:46:06.679464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"config = {\n    'n_labels': 120,\n    'batch_size': 128,\n    'lr': 1e-3,\n    'epochs': 100\n}","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.729743Z","iopub.execute_input":"2022-04-29T18:46:06.730309Z","iopub.status.idle":"2022-04-29T18:46:06.735838Z","shell.execute_reply.started":"2022-04-29T18:46:06.730258Z","shell.execute_reply":"2022-04-29T18:46:06.734440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')\n\ndef to_device(data, device):\n    \"\"\"Move tensor(s) to chosen device\"\"\"\n    if isinstance(data, (list,tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)\n    \ndevice = get_default_device()\nprint(f'Device available : {device}')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.784324Z","iopub.execute_input":"2022-04-29T18:46:06.785043Z","iopub.status.idle":"2022-04-29T18:46:06.832798Z","shell.execute_reply.started":"2022-04-29T18:46:06.785006Z","shell.execute_reply":"2022-04-29T18:46:06.831355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df, val_df = train_test_split(labels_df, test_size=0.10, random_state=42)\ntrain_df.reset_index(drop=True, inplace=True)\nval_df.reset_index(drop=True, inplace=True)\n\ntrain_dataset = DogDataset(train_df, Train_path)\nval_dataset = DogDataset(val_df, Train_path)\n\ntrain_loader = DataLoader(train_dataset, batch_size=config['batch_size'], num_workers=2, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=config['batch_size']*4, num_workers=2, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.839947Z","iopub.execute_input":"2022-04-29T18:46:06.840549Z","iopub.status.idle":"2022-04-29T18:46:06.854725Z","shell.execute_reply.started":"2022-04-29T18:46:06.840516Z","shell.execute_reply":"2022-04-29T18:46:06.853351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for data in train_loader:\n    print(data['image_data'].shape)\n    break","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:06.901534Z","iopub.execute_input":"2022-04-29T18:46:06.902160Z","iopub.status.idle":"2022-04-29T18:46:10.137417Z","shell.execute_reply.started":"2022-04-29T18:46:06.902123Z","shell.execute_reply":"2022-04-29T18:46:10.136199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def accuracy(outputs, labels):\n    _, preds = torch.max(outputs, dim=1)\n    return torch.tensor(torch.sum(preds == labels).item() / len(preds)), preds\n\n@torch.no_grad()\ndef validate_one_step(model, data):\n    for key, value in data.items():\n        data[key] = value.to(device)\n    loss, logits = model(data['image_data'], labels=data['labels'])\n    acc, _ = accuracy(logits, labels=data['labels'])\n    \n    return loss, acc\n\ndef validate_one_epoch(model, dataloader):\n    model.eval()\n    losses = []\n    accuracies = []\n    \n    for batch_idx, data in enumerate(dataloader):\n        loss, acc = validate_one_step(model, data)\n        losses.append(loss)\n        accuracies.append(acc)\n    \n    avg_loss = torch.stack(losses).mean().detach().cpu().numpy()\n    avg_acc = torch.stack(accuracies).mean().detach().cpu().numpy()\n    \n    return avg_loss, avg_acc\n\ndef train_one_step(model, data, optimizer):\n    optimizer.zero_grad()\n    for key, value in data.items():\n        data[key] = value.to(device)\n    loss, logits = model(data['image_data'], labels=data['labels'])\n    loss.backward()\n    optimizer.step()\n    \n    acc, _ = accuracy(logits, labels=data['labels'])\n    \n    return loss, acc\n\ndef train_one_epoch(model, dataloader, optimizer):\n    model.train()\n    losses = []\n    accuracies = []\n    \n    for batch_idx, data in enumerate(dataloader):\n        loss, acc = train_one_step(model, data, optimizer)\n        losses.append(loss)\n        accuracies.append(acc)\n    \n    avg_loss = torch.stack(losses).mean().detach().cpu().numpy()\n    avg_acc = torch.stack(accuracies).mean().detach().cpu().numpy()\n    \n    return avg_loss, avg_acc\n\n\ndef fit(model, train_loader, val_loader, epochs=10, lr=1e-3, opt_func=torch.optim.SGD, name=None):\n    start_time = time.time()\n    optimizer = opt_func(model.parameters(), lr=lr)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.1, patience=5, verbose=True)\n    \n    history = []\n    train_losses = []\n    train_accuracies = []\n    val_losses = []\n    val_accuracies = []\n    \n    if name is not None:\n        min_val_loss = 100\n    \n    for epoch in range(epochs):\n        # Training Phase \n        epoch_train_loss, epoch_train_acc = train_one_epoch(model, train_loader, optimizer)\n        train_losses.append(epoch_train_loss)\n        train_accuracies.append(epoch_train_acc)\n        \n        # Validation phase\n        epoch_val_loss, epoch_val_acc = validate_one_epoch(model, val_loader)\n        val_losses.append(epoch_val_loss)\n        val_accuracies.append(epoch_val_acc)\n        \n        if name is not None:\n            if epoch_val_loss < min_val_loss:\n                min_val_loss = epoch_val_acc\n                print('Lower validation loss found.')\n                name = f'epoch-{epoch}-{name}'\n                print('Saving model as ', name)\n                torch.save(model.state_dict(), name)\n                print('Saved. Continuing on...')\n        \n        print(f'Epoch [{epoch}]: Train Loss = {epoch_train_loss:.4f} | Train Acc = {epoch_train_acc:.4f} | Val Loss = {epoch_val_loss:.4f} | Val Acc = {epoch_val_acc:.4f}')\n        \n        if epoch%5 == 0:\n            print(f'Time elapsed: {(time.time() - start_time)/60:.2f} min')\n    \n        scheduler.step(epoch_train_loss)\n        \n    history = {'train_loss':train_losses, 'train_acc':train_accuracies, 'val_loss':val_losses, 'val_acc':val_accuracies}\n    return history\n\ndef plot_accuracies(history):\n    train_accuracies = history['train_acc']\n    val_accuracies = history['val_acc']\n    fig = plt.figure(figsize=(12,8))\n    plt.plot(train_accuracies, '-bx')\n    plt.plot(val_accuracies, '-rx')\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.legend(['Training', 'Validation'])\n    plt.title('Accuracy vs. No. of epochs');\n\ndef plot_losses(history):\n    train_losses = history['train_loss']\n    val_losses = history['val_loss']\n    fig = plt.figure(figsize=(12,8))\n    plt.plot(train_losses, '-bx')\n    plt.plot(val_losses, '-rx')\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.legend(['Training', 'Validation'])\n    plt.title('Loss vs. No. of epochs');\n    \ndef predict_label(data, model):\n    # Convert to a batch of 1\n    ip = to_device(data['image_data'][0].unsqueeze(0), device)\n    label = to_device(data['labels'][0].unsqueeze(0), device)\n    # Get predictions from model\n    _, p = model(ip, label)\n    # Pick index with highest probability\n    _, preds = torch.max(p, dim=1)\n    # Retrieve the class label\n    return preds\n    \ndef evaluate(model, test_loader):\n    avg_loss, avg_acc = validate_one_epoch(model, test_loader)\n    print('Accuracy  : ', avg_acc)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:10.142544Z","iopub.execute_input":"2022-04-29T18:46:10.142853Z","iopub.status.idle":"2022-04-29T18:46:10.176825Z","shell.execute_reply.started":"2022-04-29T18:46:10.142820Z","shell.execute_reply":"2022-04-29T18:46:10.175527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BasicConvNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.network = nn.Sequential(\n            nn.Conv2d(in_channels= 3, out_channels= 32, kernel_size=(9,9), stride=(1,1), padding=(4,4)), # output: 224 x 224 x 32 : 224 is the img_dimensions\n            nn.BatchNorm2d(32),\n            nn.MaxPool2d(2,2), # output: 112 x 112 x 32         \n            nn.ReLU(),\n            nn.Conv2d(in_channels= 32, out_channels= 64, kernel_size=(6,6), stride=(2,2), padding=(2,2)), # output: 56 x 56 x 64 \n            nn.BatchNorm2d(64), ##Add more conv layers maybe\n            nn.MaxPool2d(2,2), # output: 28 x 28 x 64   \n            nn.ReLU(),\n            nn.Conv2d(in_channels= 64, out_channels= 128, kernel_size=(3,3), stride=(1,1), padding=(1,1)), # output: 28 x 28 x 128 \n            nn.Flatten(),\n            nn.Dropout(0.3),\n            nn.Linear(28*28*128, 4096),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(4096, 120)\n        )\n        \n    def forward(self, xb, labels=None):\n        logits = self.network(xb)\n        out = F.log_softmax(logits, dim=1)\n        loss = 0\n        if labels is not None:\n            loss = F.nll_loss(out, labels)\n        return loss, out","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:10.178763Z","iopub.execute_input":"2022-04-29T18:46:10.179819Z","iopub.status.idle":"2022-04-29T18:46:10.195212Z","shell.execute_reply.started":"2022-04-29T18:46:10.179760Z","shell.execute_reply":"2022-04-29T18:46:10.194027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if RunAll:\n    for data in train_loader:\n        print(data['image_data'].shape)\n        loss, out = model(data['image_data'], data['labels'])\n        print(loss)\n        print(out.shape)\n        print(out[0].shape)\n        print(out[0])\n        break","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:10.198922Z","iopub.execute_input":"2022-04-29T18:46:10.199359Z","iopub.status.idle":"2022-04-29T18:46:10.209610Z","shell.execute_reply.started":"2022-04-29T18:46:10.199322Z","shell.execute_reply":"2022-04-29T18:46:10.208591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class VGGModel(nn.Module):\n    def __init__(self, out_ftrs=120, freeze=False):\n        super(VGGModel, self).__init__()\n        self.base_model = models.vgg16(pretrained=True)\n        \n        if freeze:\n            for param in self.base_model.parameters():\n                param.requires_grad = False\n                \n        num_ftrs = self.base_model.classifier[6].in_features\n        self.base_model.classifier[6] = nn.Linear(num_ftrs, out_ftrs)\n        \n\n    def forward(self, image, labels=None):\n        logits = self.base_model(image)\n        out = F.log_softmax(logits, dim=1)\n        loss = 0\n        if labels is not None:\n            loss = F.nll_loss(out, labels)\n        return loss, out\n    \n    \n    def unfreezeAll(self):\n        for param in self.base_model.parameters():\n            param.requires_grad = True","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:10.211242Z","iopub.execute_input":"2022-04-29T18:46:10.212901Z","iopub.status.idle":"2022-04-29T18:46:10.226557Z","shell.execute_reply.started":"2022-04-29T18:46:10.212865Z","shell.execute_reply":"2022-04-29T18:46:10.225460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = to_device(VGGModel(freeze=True), device)\nhistory = fit(model, train_loader, val_loader, epochs=50, lr=1e-2, opt_func=torch.optim.Adam, name='VGG_e50_adam.pth')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:10.228798Z","iopub.execute_input":"2022-04-29T18:46:10.229931Z","iopub.status.idle":"2022-04-29T18:46:14.719583Z","shell.execute_reply.started":"2022-04-29T18:46:10.229879Z","shell.execute_reply":"2022-04-29T18:46:14.718126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracies(history)\nplot_losses(history)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:14.721023Z","iopub.status.idle":"2022-04-29T18:46:14.722188Z","shell.execute_reply.started":"2022-04-29T18:46:14.721843Z","shell.execute_reply":"2022-04-29T18:46:14.721878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ResnetModel(nn.Module):\n    def __init__(self, out_ftrs=120, freeze=False):\n        super(ResnetModel, self).__init__()\n        self.base_model = models.resnet50(pretrained=True)\n        \n        if freeze:\n            for param in self.base_model.parameters():\n                param.requires_grad = False\n        \n        num_ftrs = self.base_model.fc.in_features\n        self.base_model.fc = nn.Linear(num_ftrs, out_ftrs)\n        \n\n    def forward(self, image, labels=None):\n        logits = self.base_model(image)\n        out = F.log_softmax(logits, dim=1)\n        loss = 0\n        if labels is not None:\n            loss = F.nll_loss(out, labels)\n        return loss, out\n    \n    \n    def unfreezeAll(self):\n        for param in self.base_model.parameters():\n            param.requires_grad = True","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:14.724526Z","iopub.status.idle":"2022-04-29T18:46:14.725107Z","shell.execute_reply.started":"2022-04-29T18:46:14.724806Z","shell.execute_reply":"2022-04-29T18:46:14.724838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = to_device(ResnetModel(freeze=True), device)\nhistory = fit(model, train_loader, val_loader, epochs=50, lr=1e-2, opt_func=torch.optim.Adam, name='Resnet_e50_adam.pth')","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:14.727044Z","iopub.status.idle":"2022-04-29T18:46:14.727713Z","shell.execute_reply.started":"2022-04-29T18:46:14.727357Z","shell.execute_reply":"2022-04-29T18:46:14.727390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_accuracies(history)\nplot_losses(history)","metadata":{"execution":{"iopub.status.busy":"2022-04-29T18:46:14.729730Z","iopub.status.idle":"2022-04-29T18:46:14.730344Z","shell.execute_reply.started":"2022-04-29T18:46:14.730020Z","shell.execute_reply":"2022-04-29T18:46:14.730052Z"},"trusted":true},"execution_count":null,"outputs":[]}]}