{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Summary\n\n<p style=\"font-size:17px\">\nThis notebook explores the application of Multi-Sample Dropout, as introduced in the paper <a href=\"https://arxiv.org/abs/1905.09788\">\"Multi-sample dropout for accelerated training and better generalization\"</a>, within the context of dog breed classification.</p>\n\n## Methodology:\n\n<p style=\"font-size:17px\">The methodology closely resembles that of the notebook <a href=\"https://www.kaggle.com/code/enriquecompanioni/dog-breed-classification-using-resnet-inceptionv3\">Dog Breed Classification using ResNet-InceptionV3</a>. Key differences include:</p>\n\n<ul>\n    <li><p style=\"font-size:16px\"><b>Fine-tuning Parameters:</b> The backbone models, InceptionV3 and ResNet50, undergo fine-tuning with a learning rate of 0.0001 using the Adam optimizer algorithm.</p></li>\n    <li><p style=\"font-size:16px\"><b>Dropout Rate Adjustment:</b> The dropout rate for the different classifier streams is set to 0.3.</p></li>\n    <li><p style=\"font-size:16px\"><b>Learning Rate Scheduler:</b> The learning rate scheduler is replaced with ReduceLROnPlateau to adaptively adjust the learning rate based on validation loss.</p></li>\n</ul>\n\n<p style=\"font-size:16px\">\nFeel free to adjust any hyperparameters, such as dropout rates, learning rates, or any other model settings, to potentially improve results.</p>\n\n<p style=\"font-size:17px\">\n<b>If you found this notebook helpful or simply enjoyed it, I'd greatly appreciate some upvotes! Your support keeps me motivated.</b></p>","metadata":{}},{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\n# Visualization\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# PyTorch\n\nimport torch as T\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import models\nfrom torchvision import transforms\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:28.409520Z","iopub.execute_input":"2024-04-22T21:47:28.410638Z","iopub.status.idle":"2024-04-22T21:47:28.416233Z","shell.execute_reply.started":"2024-04-22T21:47:28.410608Z","shell.execute_reply":"2024-04-22T21:47:28.415316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\", category=DeprecationWarning)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:30.031816Z","iopub.execute_input":"2024-04-22T21:47:30.032452Z","iopub.status.idle":"2024-04-22T21:47:30.036814Z","shell.execute_reply.started":"2024-04-22T21:47:30.032423Z","shell.execute_reply":"2024-04-22T21:47:30.035710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Configure GPU","metadata":{}},{"cell_type":"code","source":"device = T.device('cuda' if T.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:31.495322Z","iopub.execute_input":"2024-04-22T21:47:31.496248Z","iopub.status.idle":"2024-04-22T21:47:31.524616Z","shell.execute_reply.started":"2024-04-22T21:47:31.496213Z","shell.execute_reply":"2024-04-22T21:47:31.523739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading dataset","metadata":{}},{"cell_type":"code","source":"# Read csv file\ntrain_data = pd.read_csv(\"/kaggle/input/dog-breed-identification/labels.csv\")\n# Train data shape\nprint(f\"Train dataset shape: {train_data.shape}\")\n# Sample of the train_data DataFrame\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:33.495743Z","iopub.execute_input":"2024-04-22T21:47:33.496098Z","iopub.status.idle":"2024-04-22T21:47:33.549056Z","shell.execute_reply.started":"2024-04-22T21:47:33.496071Z","shell.execute_reply":"2024-04-22T21:47:33.548035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Read submission data","metadata":{}},{"cell_type":"code","source":"test_data = pd.DataFrame([])\nfor dirname, _, filename in os.walk('/kaggle/input/dog-breed-identification/test/'):\n    filename = pd.Series(filename)\n    test_data = pd.concat([test_data, filename], axis=0)\ntest_data.columns = ['id']\ntest_data['id'] = test_data['id'].str.replace(\".jpg\",\"\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:35.209456Z","iopub.execute_input":"2024-04-22T21:47:35.210044Z","iopub.status.idle":"2024-04-22T21:47:50.377978Z","shell.execute_reply.started":"2024-04-22T21:47:35.210015Z","shell.execute_reply":"2024-04-22T21:47:50.377072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Dataset shape\nprint(f\"Test dataset shape: {test_data.shape}\")\n# Sample of the train_data DataFrame\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.379451Z","iopub.execute_input":"2024-04-22T21:47:50.379757Z","iopub.status.idle":"2024-04-22T21:47:50.388609Z","shell.execute_reply.started":"2024-04-22T21:47:50.379732Z","shell.execute_reply":"2024-04-22T21:47:50.387581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Encoding","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\ntrain_data.loc[:,'breed'] = le.fit_transform(train_data.loc[:,'breed']) ","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.389985Z","iopub.execute_input":"2024-04-22T21:47:50.390411Z","iopub.status.idle":"2024-04-22T21:47:50.400546Z","shell.execute_reply.started":"2024-04-22T21:47:50.390378Z","shell.execute_reply":"2024-04-22T21:47:50.399593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.402742Z","iopub.execute_input":"2024-04-22T21:47:50.402999Z","iopub.status.idle":"2024-04-22T21:47:50.414041Z","shell.execute_reply.started":"2024-04-22T21:47:50.402977Z","shell.execute_reply":"2024-04-22T21:47:50.413172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dog Breed Dataset","metadata":{}},{"cell_type":"code","source":"class Dog_Breed_Dataset(Dataset):\n    \n    def __init__(self, df: pd.DataFrame, img_base_path: str, split: str, transforms = None):        \n        self.df = df\n        self.img_base_path = img_base_path\n        self.split = split\n        self.transforms = transforms\n        \n    def __getitem__(self, index):\n        # Path of the image\n        img_path = os.path.join(self.img_base_path + self.df.loc[index,'id'] + '.jpg')\n        # Read the image\n        img = Image.open(img_path)        \n        # Perform the transformations\n        if self.transforms:\n            img = self.transforms(img)        \n        if self.split != 'test':\n            y = self.df.loc[index, 'breed']                     \n            return img, y\n        else:            \n            return img\n    \n    def __len__(self):\n        return len(self.df)        ","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.415247Z","iopub.execute_input":"2024-04-22T21:47:50.415618Z","iopub.status.idle":"2024-04-22T21:47:50.423938Z","shell.execute_reply.started":"2024-04-22T21:47:50.415592Z","shell.execute_reply":"2024-04-22T21:47:50.423061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.Resize((224,224)),\n    transforms.RandomHorizontalFlip(p=0.2),\n    transforms.RandomVerticalFlip(p=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\ntest_transforms = transforms.Compose([\n    transforms.Resize((224,224)),    \n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.424958Z","iopub.execute_input":"2024-04-22T21:47:50.425229Z","iopub.status.idle":"2024-04-22T21:47:50.437228Z","shell.execute_reply.started":"2024-04-22T21:47:50.425207Z","shell.execute_reply":"2024-04-22T21:47:50.436385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Data Loaders","metadata":{}},{"cell_type":"markdown","source":"### Split the dataset","metadata":{}},{"cell_type":"code","source":"train, val = train_test_split(train_data, test_size=0.2, random_state=42, stratify=train_data['breed'])\n\ntrain = train.reset_index(drop=True)\nval = val.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.438301Z","iopub.execute_input":"2024-04-22T21:47:50.438566Z","iopub.status.idle":"2024-04-22T21:47:50.468723Z","shell.execute_reply.started":"2024-04-22T21:47:50.438544Z","shell.execute_reply":"2024-04-22T21:47:50.467759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get the data loaders","metadata":{}},{"cell_type":"code","source":"train_dataset = Dog_Breed_Dataset(\n    df=train,\n    img_base_path='/kaggle/input/dog-breed-identification/train/',\n    split='train',\n    transforms=train_transforms\n)\nvalidation_dataset = Dog_Breed_Dataset(\n    df=val,\n    img_base_path='/kaggle/input/dog-breed-identification/train/',\n    split='val',\n    transforms=test_transforms\n)\ntest_dataset = Dog_Breed_Dataset(\n    df=test_data,\n    img_base_path='/kaggle/input/dog-breed-identification/test/',\n    split='test',\n    transforms=test_transforms\n)\n\ntrain_dl = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)\nvalidation_dl = DataLoader(validation_dataset, batch_size=64, shuffle=False, num_workers=4)    \ntest_dl = DataLoader(test_dataset, batch_size=64, shuffle=False, num_workers=4)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:50.469914Z","iopub.execute_input":"2024-04-22T21:47:50.470238Z","iopub.status.idle":"2024-04-22T21:47:50.477416Z","shell.execute_reply.started":"2024-04-22T21:47:50.470213Z","shell.execute_reply":"2024-04-22T21:47:50.476526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train data length: {len(train_dl.dataset)}, Validation data length: {len(validation_dl.dataset)}, Test data length: {len(test_dl.dataset)}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:47:55.057934Z","iopub.execute_input":"2024-04-22T21:47:55.058780Z","iopub.status.idle":"2024-04-22T21:47:55.063699Z","shell.execute_reply.started":"2024-04-22T21:47:55.058741Z","shell.execute_reply":"2024-04-22T21:47:55.062766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training process","metadata":{}},{"cell_type":"markdown","source":"## Training function","metadata":{}},{"cell_type":"markdown","source":"<p style=\"font-size:15px\"><b>Note:</b> The training epoch was monitored, and the model selected as the best was found in early epoch </p>","metadata":{}},{"cell_type":"code","source":"def train_model(train_dl, val_dl, model, epochs=6):    \n    \n    # history record\n    train_acc_history = []\n    val_acc_history = []    \n    train_loss_history = []\n    val_loss_history = []\n    # Best validation loss\n    best_val_loss = 1_000_000.0    \n    # Get initial weights\n    weights = model.get_weights()\n    \n    for epoch in range(epochs):\n        print(\"=\"*20, \"Epoch: \", str(epoch), \"=\"*20)\n        \n        train_correct_pred = 0\n        val_correct_pred = 0\n        train_acc = 0\n        val_acc = 0\n        train_loss = 0\n        val_loss = 0\n        \n        # Set to training mode\n        model.train()\n        \n        for x, y in train_dl:               \n            # Convert data to Tensor            \n            x = x.clone().detach().to(device).requires_grad_(True)\n            y = y.clone().detach().long().to(device)\n            # Reset gradients\n            model.optim.zero_grad()\n            model.optim_resnet.zero_grad()\n            model.optim_inception.zero_grad()\n            # Predict\n            preds = model(x)           \n            \n            # Compute the loss            \n            loss1 = model.criterion(preds[0],y)           \n            loss2 = model.criterion(preds[1],y)           \n            loss3 = model.criterion(preds[2],y)           \n            loss4 = model.criterion(preds[3],y)\n            loss5 = model.criterion(preds[4],y)\n            \n            loss = (loss1 + loss2 + loss3 + loss4 + loss5) / 5\n            # Compute the gradients            \n            loss.backward()\n            # Update weights\n            model.optim.step()\n            model.optim_resnet.step()\n            model.optim_inception.step()\n            \n            train_loss += loss.item()                   \n            \n            pred1 = T.argmax(T.nn.functional.softmax(preds[0], dim=1), dim=1)\n            pred2 = T.argmax(T.nn.functional.softmax(preds[1], dim=1), dim=1)                                    \n            pred3 = T.argmax(T.nn.functional.softmax(preds[2], dim=1), dim=1)                                              \n            pred4 = T.argmax(T.nn.functional.softmax(preds[3], dim=1), dim=1)                                              \n            pred5 = T.argmax(T.nn.functional.softmax(preds[4], dim=1), dim=1)\n            \n            final_pred = voter(pred1, pred2, pred3, pred4, pred5)                                 \n            train_correct_pred += (final_pred.long().unsqueeze(1) == y.unsqueeze(1).cpu()).sum().item()\n        \n        train_acc = train_correct_pred / len(train_dl.dataset)\n        \n        train_acc_history.append(train_acc)\n        train_loss_history.append(train_loss)\n        \n        # Switch to evaluation mode\n        model.eval()        \n        \n        with T.no_grad():\n            for x, y in val_dl:                \n                # Convert data to Tensor                \n                x = x.clone().detach().to(device)\n                y = y.clone().detach().long().to(device)    \n                # Predict\n                preds = model(x)                \n                # Compute the loss\n                loss1 = model.criterion(preds[0],y)           \n                loss2 = model.criterion(preds[1],y)           \n                loss3 = model.criterion(preds[2],y)           \n                loss4 = model.criterion(preds[3],y)\n                loss5 = model.criterion(preds[4],y)\n\n                loss = (loss1 + loss2 + loss3 + loss4 + loss5) / 5\n                \n                pred1 = T.argmax(T.nn.functional.softmax(preds[0], dim=1), dim=1)\n                pred2 = T.argmax(T.nn.functional.softmax(preds[1], dim=1), dim=1)                                    \n                pred3 = T.argmax(T.nn.functional.softmax(preds[2], dim=1), dim=1)                                              \n                pred4 = T.argmax(T.nn.functional.softmax(preds[3], dim=1), dim=1)                                              \n                pred5 = T.argmax(T.nn.functional.softmax(preds[4], dim=1), dim=1)\n\n                final_pred = voter(pred1, pred2, pred3, pred4, pred5)\n                \n                val_correct_pred += (final_pred.long().unsqueeze(1) == y.unsqueeze(1).cpu()).sum().item()\n                \n                val_loss += loss.item()         \n            \n        model.scheduler.step(val_loss)    \n        \n        val_acc = val_correct_pred / len(val_dl.dataset)\n        \n        val_acc_history.append(val_acc)\n               \n        val_loss_history.append(val_loss)           \n        \n        if best_val_loss > val_loss:\n            best_val_loss = val_loss\n            weights = model.get_weights()\n            \n        print(\"Train acc: {:.4f} | Train Loss: {:.4f} | Validation acc: {:.4f} | Validation Loss: {:.4f}\".format(train_acc, train_loss, val_acc, val_loss))\n    model.load_weights(weights)\n    \n    model.eval()\n    # Predictions DataFrame\n    prob_preds = pd.DataFrame([])\n    with T.no_grad():\n        for x in test_dl:\n            # Convert data to Tensor            \n            x = x.clone().detach().to(device)            \n            # Predict\n            preds = model(x)  \n\n            pred1 = T.nn.functional.softmax(preds[0], dim=1)\n            pred2 = T.nn.functional.softmax(preds[1], dim=1)                                    \n            pred3 = T.nn.functional.softmax(preds[2], dim=1)                                              \n            pred4 = T.nn.functional.softmax(preds[3], dim=1)                                              \n            pred5 = T.nn.functional.softmax(preds[4], dim=1)\n\n            prob_pred = (pred1 + pred2 + pred3 + pred4 + pred5) / 5\n\n            prob_pred = prob_pred.detach().cpu().numpy()             \n            prob_pred = pd.DataFrame(prob_pred)\n\n            prob_preds = pd.concat([prob_preds, prob_pred], axis=0)            \n\n        \n    return [train_acc_history, train_loss_history, val_acc_history, val_loss_history], prob_preds","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:07:28.173487Z","iopub.execute_input":"2024-04-22T22:07:28.173930Z","iopub.status.idle":"2024-04-22T22:07:28.202059Z","shell.execute_reply.started":"2024-04-22T22:07:28.173900Z","shell.execute_reply":"2024-04-22T22:07:28.201244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Voter function","metadata":{}},{"cell_type":"code","source":"def voter(pred1, pred2, pred3, pred4, pred5):\n    \"\"\"\n        Get the final prediction through Majority Voting.        \n    \"\"\"\n    \n    final_predictions = []\n    \n    for i in range(pred1.size(0)):       \n        \n        count_df = pd.DataFrame(columns=['classes', 'count'])         \n        \n        prediction1 = pd.DataFrame({'classes': [pred1[i].item()], \"count\": [1]})\n        count_df = pd.concat([count_df, prediction1], axis=0)\n        \n        if count_df['classes'].isin([pred2[i].item()]).sum() == 1:            \n            count_df.loc[count_df['classes'].isin([pred2[i].item()]),'count'] += 1\n        else:\n            prediction2 = pd.DataFrame({'classes': [pred2[i].item()], \"count\": [1]})\n            count_df = pd.concat([count_df, prediction2], axis=0)\n        \n        if count_df['classes'].isin([pred3[i].item()]).sum() == 1:\n            count_df.loc[count_df['classes'].isin([pred3[i].item()]),'count'] += 1\n        else:\n            prediction3 = pd.DataFrame({'classes': [pred3[i].item()], \"count\": [1]})\n            count_df = pd.concat([count_df, prediction3], axis=0)\n        \n        if count_df['classes'].isin([pred4[i].item()]).sum() == 1:\n            count_df.loc[count_df['classes'].isin([pred4[i].item()]),'count'] += 1\n        else:\n            prediction4 = pd.DataFrame({'classes': [pred4[i].item()], \"count\": [1]})\n            count_df = pd.concat([count_df, prediction4], axis=0)\n            \n        if count_df['classes'].isin([pred5[i].item()]).sum() == 1:\n            count_df.loc[count_df['classes'].isin([pred5[i].item()]),'count'] += 1\n        else:\n            prediction5 = pd.DataFrame({'classes': [pred5[i].item()], \"count\": [1]})\n            count_df = pd.concat([count_df, prediction5], axis=0)            \n        \n        if len(count_df.loc[count_df['count'] == count_df['count'].max(), 'classes'].values) > 1:\n            pred = count_df.loc[count_df['count'] == count_df['count'].max(), 'classes'].values\n            pred = int(pred[0])\n        else:\n            pred = int(count_df.loc[count_df['count'] == count_df['count'].max(), 'classes'].values)\n        final_predictions.append(pred)\n    return T.tensor(final_predictions)    ","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:12:00.206427Z","iopub.execute_input":"2024-04-22T22:12:00.206766Z","iopub.status.idle":"2024-04-22T22:12:00.222255Z","shell.execute_reply.started":"2024-04-22T22:12:00.206742Z","shell.execute_reply":"2024-04-22T22:12:00.221293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"markdown","source":"Inception model","metadata":{}},{"cell_type":"code","source":"inception = models.inception_v3(pretrained=True)\n\ninception_model = nn.Sequential(\n    inception.Conv2d_1a_3x3,\n    inception.Conv2d_2a_3x3,\n    inception.Conv2d_2b_3x3,\n    inception.maxpool1,\n    inception.Conv2d_3b_1x1,\n    inception.Conv2d_4a_3x3,\n    inception.maxpool2,\n    inception.Mixed_5b,\n    inception.Mixed_5c,\n    inception.Mixed_5d,\n    inception.Mixed_6a,\n    inception.Mixed_6b,\n    inception.Mixed_6c,\n    inception.Mixed_6d,\n    inception.Mixed_6e,\n    inception.Mixed_7a,\n    inception.Mixed_7b,\n    inception.Mixed_7c,\n    inception.avgpool\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:48:13.067586Z","iopub.execute_input":"2024-04-22T21:48:13.068343Z","iopub.status.idle":"2024-04-22T21:48:14.889992Z","shell.execute_reply.started":"2024-04-22T21:48:13.068309Z","shell.execute_reply":"2024-04-22T21:48:14.889142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Resnet50 model","metadata":{}},{"cell_type":"code","source":"resnet50 = models.resnet50(pretrained=True)\n\nresnet50_model = nn.Sequential(\n    resnet50.conv1,\n    resnet50.bn1,\n    resnet50.relu,\n    resnet50.maxpool,\n    resnet50.layer1,\n    resnet50.layer2,\n    resnet50.layer3,\n    resnet50.layer4,\n    resnet50.avgpool\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:48:15.353825Z","iopub.execute_input":"2024-04-22T21:48:15.354187Z","iopub.status.idle":"2024-04-22T21:48:16.618170Z","shell.execute_reply.started":"2024-04-22T21:48:15.354160Z","shell.execute_reply":"2024-04-22T21:48:16.617333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model class","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, inception_model, resnet50_model):\n        super(Model,self).__init__()\n        \n        self.inception_model = inception_model\n        self.resnet50_model = resnet50_model        \n        \n        self.output = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(4096,120)            \n        )\n        \n        self.to(device)\n        # Optimizer \n        self.optim = T.optim.SGD(self.output.parameters(), lr=0.005, momentum=0.9)\n        self.optim_resnet = T.optim.Adam(self.resnet50_model.parameters(), lr=0.0001)\n        self.optim_inception = T.optim.Adam(self.inception_model.parameters(), lr=0.0001)\n        # Loss\n        self.criterion = T.nn.CrossEntropyLoss()\n        # Scheduler\n        self.scheduler = ReduceLROnPlateau(self.optim, mode='min', factor=0.1, patience=5)        \n        \n    def forward(self, x):\n        X1 = self.inception_model(x)\n        X2 = self.resnet50_model(x)\n        \n        X1 = X1.view(X1.size(0), -1)\n        X2 = X2.view(X2.size(0), -1)\n       \n        X = T.cat([X1, X2], dim=1)\n        \n        P1 = self.output(X)\n        P2 = self.output(X)\n        P3 = self.output(X)\n        P4 = self.output(X)\n        P5 = self.output(X)\n        \n        return [P1, P2, P3, P4, P5]\n    \n    def get_weights(self):\n        return self.state_dict()\n    \n    def load_weights(self, weights):\n        self.load_state_dict(weights)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:48:18.699688Z","iopub.execute_input":"2024-04-22T21:48:18.700028Z","iopub.status.idle":"2024-04-22T21:48:18.710871Z","shell.execute_reply.started":"2024-04-22T21:48:18.700003Z","shell.execute_reply":"2024-04-22T21:48:18.709849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training model","metadata":{}},{"cell_type":"code","source":"model = Model(inception_model, resnet50_model)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:48:31.217275Z","iopub.execute_input":"2024-04-22T21:48:31.217964Z","iopub.status.idle":"2024-04-22T21:48:31.449748Z","shell.execute_reply.started":"2024-04-22T21:48:31.217935Z","shell.execute_reply":"2024-04-22T21:48:31.448980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history, test_preds = train_model(train_dl, validation_dl, model)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T21:48:37.406594Z","iopub.execute_input":"2024-04-22T21:48:37.406939Z","iopub.status.idle":"2024-04-22T22:04:59.312330Z","shell.execute_reply.started":"2024-04-22T21:48:37.406913Z","shell.execute_reply":"2024-04-22T22:04:59.311026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training and Validation Results\nfig, axs = plt.subplots(1,2, figsize=(15,6))\naxs[0].plot(range(6), history[0], label=\"Training accuracy\")\naxs[0].plot(range(6), history[2], label=\"Validation accuracy\")\naxs[0].set_xlabel('Epoch')\naxs[0].set_ylabel('Accuracy')\naxs[0].grid(True)\n\naxs[1].plot(range(6), history[1], label=\"Training Loss\")\naxs[1].plot(range(6), history[3], label=\"Validation Loss\")\naxs[1].set_xlabel('Epoch')\naxs[1].set_ylabel('Loss')\naxs[1].grid(True)\n\nplt.suptitle(\"Training and Validation Results of Model\")\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:07.681162Z","iopub.execute_input":"2024-04-22T22:05:07.681557Z","iopub.status.idle":"2024-04-22T22:05:08.124665Z","shell.execute_reply.started":"2024-04-22T22:05:07.681519Z","shell.execute_reply":"2024-04-22T22:05:08.123848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test_preds.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:15.417208Z","iopub.execute_input":"2024-04-22T22:05:15.417576Z","iopub.status.idle":"2024-04-22T22:05:15.446604Z","shell.execute_reply.started":"2024-04-22T22:05:15.417548Z","shell.execute_reply":"2024-04-22T22:05:15.445748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set columns to breed names\nnum_classes = []\nfor num_class in test_preds.columns:\n    num_classes.append(num_class)\n\nnum_classes = np.array(num_classes)\nnum_classes = le.inverse_transform(num_classes)\ntest_preds.columns = list(num_classes)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:17.072975Z","iopub.execute_input":"2024-04-22T22:05:17.073341Z","iopub.status.idle":"2024-04-22T22:05:17.079831Z","shell.execute_reply.started":"2024-04-22T22:05:17.073314Z","shell.execute_reply":"2024-04-22T22:05:17.078872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:19.480097Z","iopub.execute_input":"2024-04-22T22:05:19.480758Z","iopub.status.idle":"2024-04-22T22:05:19.511348Z","shell.execute_reply.started":"2024-04-22T22:05:19.480725Z","shell.execute_reply":"2024-04-22T22:05:19.510366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set id column\ntest_preds = test_preds.reset_index(drop=True)\nids = test_data.loc[:,'id']\ntest_preds = pd.concat([ids, test_preds], axis=1)\ntest_preds.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:21.568960Z","iopub.execute_input":"2024-04-22T22:05:21.569812Z","iopub.status.idle":"2024-04-22T22:05:21.604047Z","shell.execute_reply.started":"2024-04-22T22:05:21.569780Z","shell.execute_reply":"2024-04-22T22:05:21.603192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T22:05:25.783519Z","iopub.execute_input":"2024-04-22T22:05:25.783869Z","iopub.status.idle":"2024-04-22T22:05:27.705569Z","shell.execute_reply.started":"2024-04-22T22:05:25.783842Z","shell.execute_reply":"2024-04-22T22:05:27.704561Z"},"trusted":true},"execution_count":null,"outputs":[]}]}