{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-10-01T17:41:26.136100Z","iopub.execute_input":"2023-10-01T17:41:26.136833Z","iopub.status.idle":"2023-10-01T17:42:05.563576Z","shell.execute_reply.started":"2023-10-01T17:41:26.136798Z","shell.execute_reply":"2023-10-01T17:42:05.562683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"Contents:\n    1. Data preprocessing\n        1.1 Prepare images dataframe <id, tensor> and labels dataframe <id, label>\n        1.2 Create custom train and test datasets - match labels with dog images\n        1.3 Create DataLoaders\n    2. Build model\n    3. Train model\n    4. Try pretrained model","metadata":{}},{"cell_type":"code","source":"## 1.1 Prepare images dataframe <id, tensor> and labels dataframe <id, label>\n\nimport csv\nfrom sklearn.model_selection import train_test_split\n\n\ntraining_data_path = \"/kaggle/input/dog-breed-identification/train/\"\nsubmission_data_path = \"/kaggle/input/dog-breed-identification/test/\"\nlabels_path = \"/kaggle/input/dog-breed-identification/labels.csv\"\n\n\nlabels_df = pd.read_csv(labels_path)\n\nbreeds = sorted(labels_df['breed'].unique())\nbreed_to_label = {breed: i for i, breed in enumerate(breeds)}\n\n\nNUM_CLASSES = len(breeds)\n\n\n# SPLIT TRAINING AND TEST DATA\ntraining_labels, test_labels = train_test_split(labels_df, train_size=0.8, random_state=42, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:05.565428Z","iopub.execute_input":"2023-10-01T17:42:05.565956Z","iopub.status.idle":"2023-10-01T17:42:06.113406Z","shell.execute_reply.started":"2023-10-01T17:42:05.565917Z","shell.execute_reply":"2023-10-01T17:42:06.112421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot training and test dataset breeds distribution\nprint(training_labels.breed.value_counts())\ntraining_labels.breed.value_counts().plot.bar(figsize=(15, 15))","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:06.114828Z","iopub.execute_input":"2023-10-01T17:42:06.115205Z","iopub.status.idle":"2023-10-01T17:42:07.547788Z","shell.execute_reply.started":"2023-10-01T17:42:06.115172Z","shell.execute_reply":"2023-10-01T17:42:07.546838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_labels.breed.value_counts())\ntest_labels.breed.value_counts().plot.bar(figsize=(15, 15))","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:07.550162Z","iopub.execute_input":"2023-10-01T17:42:07.551075Z","iopub.status.idle":"2023-10-01T17:42:08.846887Z","shell.execute_reply.started":"2023-10-01T17:42:07.551041Z","shell.execute_reply":"2023-10-01T17:42:08.845938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image\nimport torch\nfrom torch import nn\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset\n\nclass DogBreedsDataset(Dataset):\n    def __init__(self, img_transforms, images_root_path, labels:pd.DataFrame = None):\n        self.labels = labels\n        self.root = images_root_path\n        self.transforms = img_transforms\n        \n    def __len__(self):\n        if self.labels is None:\n            return len(os.listdir(self.root))\n        else:\n            return len(self.labels)\n    \n    def __getitem__(self, idx):\n        if self.labels is None:\n            img_id = os.listdir(self.root)[idx]\n            img = Image.open(f'{self.root}{img_id}')\n            return self.transforms(img).to(dtype=torch.float32), img_id.strip(\".jpg\")\n        else:\n            img_id = self.labels.iloc[idx].id\n            img_label = self.labels.iloc[idx].breed\n            img = Image.open(f'{self.root}{img_id}.jpg')\n            img_tensor = self.transforms(img).to(dtype=torch.float32)\n            encoded_label = nn.functional.one_hot(torch.tensor(breed_to_label[img_label]).to(torch.int64), NUM_CLASSES)\n#             encoded_label = torch.tensor(breed_to_label[img_label]).to(torch.int64)\n            return img_tensor, encoded_label\n","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:08.848400Z","iopub.execute_input":"2023-10-01T17:42:08.848988Z","iopub.status.idle":"2023-10-01T17:42:12.297778Z","shell.execute_reply.started":"2023-10-01T17:42:08.848957Z","shell.execute_reply":"2023-10-01T17:42:12.296856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE DATASETS\n\nIMG_SIZE = 224\n\ntraining_transforms = transforms.Compose([\n    transforms.Resize(IMG_SIZE),\n    transforms.CenterCrop(IMG_SIZE),\n    transforms.ToTensor(),\n    transforms.Normalize(\n            mean=[0.485, 0.456, 0.406],\n            std=[0.229, 0.224, 0.225]\n    )])\n\ntrain_dataset = DogBreedsDataset(\n#     img_transforms=transforms.Compose([\n#         transforms.Resize((IMG_SIZE, IMG_SIZE)),\n#         transforms.PILToTensor()\n#     ]),\n    img_transforms=training_transforms,\n    images_root_path=training_data_path, \n    labels=training_labels\n)\n\ntest_dataset = DogBreedsDataset(\n#     img_transforms=transforms.Compose([\n#         transforms.Resize((IMG_SIZE, IMG_SIZE)),\n#         transforms.PILToTensor()\n#     ]),\n    img_transforms=training_transforms,\n    images_root_path=training_data_path, \n    labels=test_labels\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:12.299093Z","iopub.execute_input":"2023-10-01T17:42:12.299611Z","iopub.status.idle":"2023-10-01T17:42:12.305914Z","shell.execute_reply.started":"2023-10-01T17:42:12.299576Z","shell.execute_reply":"2023-10-01T17:42:12.304962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 1.3 Create DataLoaders\n\nfrom torch.utils.data import DataLoader\n\nBATCH_SIZE = 128\n\ntrain_dataloader = DataLoader(\n    train_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=True\n)\n\ntest_dataloader = DataLoader(\n    test_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False\n)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:12.307040Z","iopub.execute_input":"2023-10-01T17:42:12.308075Z","iopub.status.idle":"2023-10-01T17:42:12.323744Z","shell.execute_reply.started":"2023-10-01T17:42:12.308043Z","shell.execute_reply":"2023-10-01T17:42:12.322802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2. Build model\n\nclass DogBreedClassificationModel(nn.Module):\n    \"\"\"\n    Model architecture copying TinyVGG from: \n    https://poloclub.github.io/cnn-explainer/\n    \"\"\"\n    def __init__(self, input_shape: int, hidden_units: int, output_shape: int):\n        super().__init__()\n        self.block_1 = nn.Sequential(\n            nn.Conv2d(in_channels=input_shape, \n                      out_channels=hidden_units, \n                      kernel_size=3, # how big is the square that's going over the image?\n                      stride=1, # default\n                      padding=1),# options = \"valid\" (no padding) or \"same\" (output has same shape as input) or int for specific number \n            nn.ReLU(),\n            nn.Conv2d(in_channels=hidden_units, \n                      out_channels=hidden_units,\n                      kernel_size=3,\n                      stride=1,\n                      padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=2,\n                         stride=2) # default stride value is same as kernel_size\n        )\n        self.block_2 = nn.Sequential(\n            nn.Conv2d(hidden_units, hidden_units, 3, padding=1),\n            nn.ReLU(),\n            nn.Conv2d(hidden_units, hidden_units, 3, padding=1),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        self.classifier = nn.Sequential(\n            nn.Flatten(),\n            nn.Linear(in_features=int(hidden_units*(IMG_SIZE/4)*(IMG_SIZE/4)), out_features=output_shape)\n        )\n    \n    def forward(self, x: torch.Tensor):\n        x = self.block_1(x)\n        x = self.block_2(x)\n        x = self.classifier(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:12.325321Z","iopub.execute_input":"2023-10-01T17:42:12.326458Z","iopub.status.idle":"2023-10-01T17:42:12.336674Z","shell.execute_reply.started":"2023-10-01T17:42:12.326419Z","shell.execute_reply":"2023-10-01T17:42:12.335733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    import torchmetrics\nexcept:\n    !pip install torchmetrics\n    import torchmetrics","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:12.338295Z","iopub.execute_input":"2023-10-01T17:42:12.339007Z","iopub.status.idle":"2023-10-01T17:42:22.026547Z","shell.execute_reply.started":"2023-10-01T17:42:12.338955Z","shell.execute_reply":"2023-10-01T17:42:22.025449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import cuda\n\ndevice = 'cuda' if cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:22.029764Z","iopub.execute_input":"2023-10-01T17:42:22.030754Z","iopub.status.idle":"2023-10-01T17:42:22.066936Z","shell.execute_reply.started":"2023-10-01T17:42:22.030719Z","shell.execute_reply":"2023-10-01T17:42:22.065579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_step(data_loader, model, loss_fn, optimizer, accuracy_fn, device, lr_scheduler):\n    train_loss, train_acc = 0, 0\n    model.to(device)\n    loss_fn.to(device)\n    accuracy_fn.to(device)\n    model.train()\n    for batch, (X, y) in enumerate(data_loader):\n#         print(f'Train step: Batch {batch}, X.shape {X.shape}, y.shape {y.shape}')\n        # Send data to GPU\n        X, y = X.to(device), y.to(device)\n\n        # 1. Forward pass\n        y_logits = model(X)\n        loss = loss_fn(y_logits, y.to(torch.float32))\n        train_loss += loss\n#         train_acc += accuracy_fn(y_logits.softmax(dim=1), y.to(torch.float32))\n\n        # 3. Optimizer zero grad\n        optimizer.zero_grad()\n\n        # 4. Loss backward\n        loss.backward()\n\n        # 5. Optimizer step\n        optimizer.step()\n    \n    if lr_scheduler is not None:\n        lr_scheduler.step(train_loss)\n\n    # Calculate loss and accuracy per epoch and print out what's happening\n    train_loss /= len(data_loader)\n    train_acc /= len(data_loader)\n    print(f\"Train loss: {train_loss:.5f}\")#| Train accuracy: {train_acc:.2f}%\")\n    \n\ndef test_step(data_loader, model, loss_fn, accuracy_fn, device, confusion_matrix):\n    test_loss, test_acc = 0, 0\n    model.to(device)\n    loss_fn.to(device)\n    accuracy_fn.to(device)\n    model.eval() # put model in eval mode\n    # Turn on inference context manager\n    with torch.inference_mode(): \n        for X, y in data_loader:\n#             print(f'Test step: X.shape {X.shape}, y.shape {y.shape}')\n            # Send data to GPU\n            X, y = X.to(device), y.to(device)\n            \n            # 1. Forward pass\n            test_logits = model(X)\n            \n            # 2. Calculate loss and accuracy\n            test_loss += loss_fn(test_logits, y.to(torch.float32))\n#             test_acc += accuracy_fn(test_logits.softmax(dim=1), y.to(torch.float32))\n            confusion_matrix.update(test_logits.softmax(dim=1), y)\n        \n        # Adjust metrics and print out\n        test_loss /= len(data_loader)\n        test_acc /= len(data_loader)\n        print(f\"Test loss: {test_loss:.5f}\")# | Test accuracy: {test_acc:.2f}%\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:22.068916Z","iopub.execute_input":"2023-10-01T17:42:22.069293Z","iopub.status.idle":"2023-10-01T17:42:22.081493Z","shell.execute_reply.started":"2023-10-01T17:42:22.069261Z","shell.execute_reply":"2023-10-01T17:42:22.080491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3. Train model\nfrom tqdm.auto import tqdm\n\nEPOCHS = 10\nHIDDEN_UNITS = 20\nLEARNING_RATE = 0.001\nMOMENTUM = 0.9\nCLASSES = len(breeds)\n\nmodel = DogBreedClassificationModel(\n    input_shape=3, # color channels\n    hidden_units=HIDDEN_UNITS,\n    output_shape=CLASSES # classes count\n)\n\nloss_fn = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(params=model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM)\n# lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9)\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\nplateau_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')\naccuracy_fn = torchmetrics.classification.MulticlassAccuracy(num_classes=CLASSES)\n\n# for epoch in tqdm(range(EPOCHS)):\n#     print(f\"Epoch: {epoch}\\n---------\")\n#     train_step(\n#         train_dataloader, \n#         model, \n#         loss_fn, \n#         optimizer, \n#         accuracy_fn, \n#         device,\n#         lr_scheduler\n#     )\n    \n#     test_step(\n#         test_dataloader, \n#         model, \n#         loss_fn,\n#         accuracy_fn, \n#         device\n#     )\n#     print(\"End epoch\")","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:22.083172Z","iopub.execute_input":"2023-10-01T17:42:22.083944Z","iopub.status.idle":"2023-10-01T17:42:22.204026Z","shell.execute_reply.started":"2023-10-01T17:42:22.083911Z","shell.execute_reply":"2023-10-01T17:42:22.202933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 4. Try pretrained model\n## 4.1 Load pretrained resnet50 model\nfrom torchvision import models\nfrom tqdm.auto import tqdm\nfrom torchmetrics.classification import MulticlassConfusionMatrix\n\nmodel_transfer = models.resnet50(pretrained=True)\n\n# Freeze training for all \"features\" layers\nfor param in model_transfer.parameters():\n    param.requires_grad = False\n    \n# replace the last fully connected layer with a Linnear layer output\nin_features = model_transfer.fc.in_features\nmodel_transfer.fc = nn.Linear(in_features, CLASSES)\n\n\nmodel_transfer = model_transfer.to(device)\n\nLEARNING_RATE = 0.02\n\n## 4.2 Setup loss fn and optimizer\ncriterion_transfer = nn.CrossEntropyLoss()\nmodel_transfer_grad_paramaters = filter(lambda p: p.requires_grad, model_transfer.parameters())\noptimizer_transfer = torch.optim.SGD(model_transfer_grad_paramaters, lr=LEARNING_RATE, momentum=MOMENTUM)\ntransfer_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer_transfer, gamma=0.9)\n\nconfusion_matrix = MulticlassConfusionMatrix(num_classes=CLASSES).to(device)\n\n\n## 4.3 Train loaded model\n\nCLASSES = len(breeds)\nacc = torchmetrics.classification.MulticlassAccuracy(num_classes=CLASSES)\n\n\nfor epoch in tqdm(range(EPOCHS)):\n    print(f\"Epoch: {epoch}\\n---------\")\n    train_step(\n        train_dataloader, \n        model_transfer, \n        criterion_transfer, \n        optimizer_transfer, \n        acc, \n        device,\n        transfer_lr_scheduler\n    )\n    test_step(\n        test_dataloader, \n        model_transfer, \n        criterion_transfer,\n        acc, \n        device,\n        confusion_matrix\n    )\n    print(\"End epoch\")\n\nconfusion_matrix#.plot() # -> try to resize or return as a csv report","metadata":{"execution":{"iopub.status.busy":"2023-10-01T17:42:22.205491Z","iopub.execute_input":"2023-10-01T17:42:22.205840Z","iopub.status.idle":"2023-10-01T17:55:47.907965Z","shell.execute_reply.started":"2023-10-01T17:42:22.205808Z","shell.execute_reply":"2023-10-01T17:55:47.907031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare submission\nimport torchvision\n\nsubmission_dataset = DogBreedsDataset(\n    img_transforms=transforms.Compose([\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.PILToTensor()\n    ]),\n    images_root_path=submission_data_path\n)\n\nsubmission_dataloader = DataLoader(\n    submission_dataset, \n    batch_size=BATCH_SIZE, \n    shuffle=False\n)\n\n\ndef prepare_sub_with_dataloader(model, dataloader):\n    model.eval()\n    model.to(device)\n    \n    column_names = breeds.copy()\n    column_names.insert(0, 'id')\n    \n    submission_df = pd.DataFrame(columns=column_names)\n    \n    with torch.inference_mode():\n        for X, image_ids in dataloader:\n            X = X.to(device)\n            sub_preds = model(X).softmax(dim=1).tolist()\n            \n            for idx, image_id in enumerate(image_ids):\n                item_pred = sub_preds[idx]\n                item_pred.insert(0, image_id)\n                submission_df.loc[len(submission_df)] = item_pred\n    return submission_df","metadata":{"execution":{"iopub.status.busy":"2023-10-01T18:48:17.561435Z","iopub.execute_input":"2023-10-01T18:48:17.561813Z","iopub.status.idle":"2023-10-01T18:48:17.570001Z","shell.execute_reply.started":"2023-10-01T18:48:17.561783Z","shell.execute_reply":"2023-10-01T18:48:17.568676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = prepare_sub_with_dataloader(model_transfer, submission_dataloader)\nsubmission_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-01T18:48:23.066672Z","iopub.execute_input":"2023-10-01T18:48:23.067038Z","iopub.status.idle":"2023-10-01T18:52:26.227712Z","shell.execute_reply.started":"2023-10-01T18:48:23.067001Z","shell.execute_reply":"2023-10-01T18:52:26.226446Z"},"trusted":true},"execution_count":null,"outputs":[]}]}