{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"}],"dockerImageVersionId":30683,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Libraries","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\n# Visualization\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# PyTorch\n\nimport torch as T\nfrom torch import nn\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision import models\nfrom torchvision import transforms","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:23.3441Z","iopub.execute_input":"2024-04-23T06:32:23.344782Z","iopub.status.idle":"2024-04-23T06:32:23.350421Z","shell.execute_reply.started":"2024-04-23T06:32:23.344753Z","shell.execute_reply":"2024-04-23T06:32:23.349567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Configure GPU","metadata":{}},{"cell_type":"code","source":"device = T.device('cuda' if T.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:25.269634Z","iopub.execute_input":"2024-04-23T06:32:25.270018Z","iopub.status.idle":"2024-04-23T06:32:25.298727Z","shell.execute_reply.started":"2024-04-23T06:32:25.269981Z","shell.execute_reply":"2024-04-23T06:32:25.297534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading dataset","metadata":{}},{"cell_type":"code","source":"# Read csv file\ntrain_data = pd.read_csv(\"/kaggle/input/dog-breed-identification/labels.csv\")\n# Train data shape\nprint(f\"Train dataset shape: {train_data.shape}\")\n# Sample of the train_data DataFrame\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:25.638726Z","iopub.execute_input":"2024-04-23T06:32:25.639791Z","iopub.status.idle":"2024-04-23T06:32:25.693517Z","shell.execute_reply.started":"2024-04-23T06:32:25.639748Z","shell.execute_reply":"2024-04-23T06:32:25.692608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Distribution of the breed classes","metadata":{}},{"cell_type":"code","source":"breed_classes = train_data.breed.value_counts().reset_index()\nplt.figure(figsize=(20,8))\nsns.barplot(breed_classes, x='breed', y='count', palette=\"flare\")\nplt.xticks(rotation=90)\nplt.title(\"Distribution of the breed classes\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:26.812871Z","iopub.execute_input":"2024-04-23T06:32:26.813702Z","iopub.status.idle":"2024-04-23T06:32:28.165176Z","shell.execute_reply.started":"2024-04-23T06:32:26.813669Z","shell.execute_reply":"2024-04-23T06:32:28.164237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"breed_classes['count'].describe()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:28.167046Z","iopub.execute_input":"2024-04-23T06:32:28.167448Z","iopub.status.idle":"2024-04-23T06:32:28.179679Z","shell.execute_reply.started":"2024-04-23T06:32:28.16742Z","shell.execute_reply":"2024-04-23T06:32:28.178716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Confirm the 120 dog breed classes","metadata":{}},{"cell_type":"code","source":"breed_classes['breed'].nunique()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:28.180839Z","iopub.execute_input":"2024-04-23T06:32:28.181104Z","iopub.status.idle":"2024-04-23T06:32:28.190489Z","shell.execute_reply.started":"2024-04-23T06:32:28.181082Z","shell.execute_reply":"2024-04-23T06:32:28.189438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Label Encoding","metadata":{}},{"cell_type":"code","source":"le = LabelEncoder()\ntrain_data['breed'] = le.fit_transform(train_data.loc[:,'breed']) ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:28.192137Z","iopub.execute_input":"2024-04-23T06:32:28.192414Z","iopub.status.idle":"2024-04-23T06:32:28.202853Z","shell.execute_reply.started":"2024-04-23T06:32:28.192392Z","shell.execute_reply":"2024-04-23T06:32:28.202052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = dict(zip(le.classes_, le.transform(le.classes_)))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:28.402473Z","iopub.execute_input":"2024-04-23T06:32:28.403347Z","iopub.status.idle":"2024-04-23T06:32:28.407843Z","shell.execute_reply.started":"2024-04-23T06:32:28.403314Z","shell.execute_reply":"2024-04-23T06:32:28.406809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dog Breed Dataset","metadata":{}},{"cell_type":"code","source":"class Dog_Breed_Dataset(Dataset):\n    \n    def __init__(self, df: pd.DataFrame, img_base_path: str, split: str, transforms = None):        \n        self.df = df\n        self.img_base_path = img_base_path\n        self.split = split\n        self.transforms = transforms\n        \n    def __getitem__(self, index):\n        # Path of the image\n        img_path = os.path.join(self.img_base_path + self.df.loc[index,'id'] + '.jpg')\n        # Read the image\n        img = Image.open(img_path)        \n        # Perform the transformations\n        if self.transforms:\n            img = self.transforms(img)\n        \n        if self.split != 'test':\n            y = self.df.loc[index, 'breed']                     \n            return img, y\n        else:            \n            return img\n    \n    def __len__(self):\n        return len(self.df)        ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:28.95946Z","iopub.execute_input":"2024-04-23T06:32:28.959816Z","iopub.status.idle":"2024-04-23T06:32:28.96749Z","shell.execute_reply.started":"2024-04-23T06:32:28.959788Z","shell.execute_reply":"2024-04-23T06:32:28.966545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Augmentation","metadata":{}},{"cell_type":"code","source":"train_transforms = transforms.Compose([\n    transforms.Resize((224,224)),\n    transforms.RandomHorizontalFlip(p=0.2),\n    transforms.RandomVerticalFlip(p=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\ntest_transforms = transforms.Compose([\n    transforms.Resize((224,224)),    \n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:29.732441Z","iopub.execute_input":"2024-04-23T06:32:29.733145Z","iopub.status.idle":"2024-04-23T06:32:29.739477Z","shell.execute_reply.started":"2024-04-23T06:32:29.733114Z","shell.execute_reply":"2024-04-23T06:32:29.73845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get Data Loaders","metadata":{}},{"cell_type":"markdown","source":"### Split the dataset","metadata":{}},{"cell_type":"code","source":"train, val = train_test_split(train_data, test_size=0.2, random_state=42, stratify=train_data['breed'])\n\ntrain = train.reset_index(drop=True)\nval = val.reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:44.967547Z","iopub.execute_input":"2024-04-23T06:32:44.968539Z","iopub.status.idle":"2024-04-23T06:32:44.986766Z","shell.execute_reply.started":"2024-04-23T06:32:44.968503Z","shell.execute_reply":"2024-04-23T06:32:44.985742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get the data loaders","metadata":{}},{"cell_type":"code","source":"train_dataset = Dog_Breed_Dataset(\n    df=train,\n    img_base_path='/kaggle/input/dog-breed-identification/train/',\n    split='train',\n    transforms=train_transforms\n)\nvalidation_dataset = Dog_Breed_Dataset(\n    df=val,\n    img_base_path='/kaggle/input/dog-breed-identification/train/',\n    split='val',\n    transforms=test_transforms\n)\n\ntrain_dl = DataLoader(train_dataset, batch_size=64, shuffle=True, num_workers=4)\nvalidation_dl = DataLoader(validation_dataset, batch_size=64, shuffle=False, num_workers=4)    ","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:45.886847Z","iopub.execute_input":"2024-04-23T06:32:45.887552Z","iopub.status.idle":"2024-04-23T06:32:45.893514Z","shell.execute_reply.started":"2024-04-23T06:32:45.887519Z","shell.execute_reply":"2024-04-23T06:32:45.892532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Train data length: {len(train_dl.dataset)}, Validation data length: {len(validation_dl.dataset)}\")","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:46.89731Z","iopub.execute_input":"2024-04-23T06:32:46.897909Z","iopub.status.idle":"2024-04-23T06:32:46.902798Z","shell.execute_reply.started":"2024-04-23T06:32:46.897876Z","shell.execute_reply":"2024-04-23T06:32:46.901822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def imshow(axis, inp):\n#     \"\"\"Denormalize and show\"\"\"\n#     inp = inp.numpy().transpose((1, 2, 0))\n#     mean = np.array([0.485, 0.456, 0.406])\n#     std = np.array([0.229, 0.224, 0.225])\n#     inp = std * inp + mean\n#     axis.imshow(inp)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:48.16524Z","iopub.execute_input":"2024-04-23T06:32:48.165928Z","iopub.status.idle":"2024-04-23T06:32:48.169827Z","shell.execute_reply.started":"2024-04-23T06:32:48.165897Z","shell.execute_reply":"2024-04-23T06:32:48.168907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from mpl_toolkits.axes_grid1 import ImageGrid\n\n# img, label = next(iter(train_dataset))\n# print(img.size(), label)\n\n# fig = plt.figure(1, figsize=(16, 12))\n# grid = ImageGrid(fig, 111, nrows_ncols=(3, 4), axes_pad=0.05)    \n\n# for i in range(img.size()[0]):\n#     ax = grid[i]\n#     imshow(ax, img[i])","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:32:48.399293Z","iopub.execute_input":"2024-04-23T06:32:48.399574Z","iopub.status.idle":"2024-04-23T06:32:48.403876Z","shell.execute_reply.started":"2024-04-23T06:32:48.399552Z","shell.execute_reply":"2024-04-23T06:32:48.402981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training process","metadata":{}},{"cell_type":"markdown","source":"## Training function","metadata":{}},{"cell_type":"code","source":"def train_model(train_dl, val_dl, model, epochs=50):    \n    \n    train_acc_history = []\n    val_acc_history = []\n    train_loss_history = []\n    val_loss_history = []\n    # Best validation accuracy\n    best_val_loss = 1_000_000.0    \n    # Get initial weights\n    weights = model.get_weights()\n    \n    for epoch in range(epochs):\n        print(\"=\"*20, \"Epoch: \", str(epoch), \"=\"*20)\n        \n        train_correct_pred = 0\n        val_correct_pred = 0\n        train_acc = 0\n        val_acc = 0\n        train_loss = 0\n        val_loss = 0\n        \n        # Set to training mode\n        model.train()\n        \n        for x, y in train_dl:               \n            # Convert data to Tensor            \n            x = x.clone().detach().to(device).requires_grad_(True)\n            y = y.clone().detach().long().to(device)\n            # Reset gradients\n            model.optim.zero_grad()\n            # Predict\n            preds = model(x)            \n            \n            # Compute the loss            \n            loss = model.criterion(preds,y)            \n            \n            # Compute the gradients            \n            loss.backward()\n            # Update weights\n            model.optim.step()\n            # Count the correct predictions\n            preds = T.argmax(preds, dim=1)           \n            train_correct_pred += (preds.long().unsqueeze(1) == y.unsqueeze(1)).sum().item()\n            \n            train_loss += loss.item()           \n        \n        train_acc = train_correct_pred / len(train_dl.dataset)\n        \n        train_acc_history.append(train_acc)\n               \n        train_loss_history.append(train_loss)\n        \n        # Switch to evaluation mode\n        model.eval()        \n        \n        with T.no_grad():\n            for x, y in val_dl:                \n                # Convert data to Tensor                \n                x = x.clone().detach().to(device)\n                y = y.clone().detach().long().to(device)    \n                # Predict\n                preds = model(x)                \n                # Compute the loss\n                loss = model.criterion(preds,y)                                         \n                \n                val_loss += loss.item()                \n                # Count the correct predictions\n                preds = T.argmax(preds, dim=1)\n                \n                val_correct_pred += (preds.long().unsqueeze(1) == y.unsqueeze(1)).sum().item() \n                \n        model.scheduler.step()       \n        \n        val_acc = val_correct_pred / len(val_dl.dataset)\n        \n        val_acc_history.append(val_acc)\n        val_loss_history.append(val_loss)           \n        # Save the weights of the best model\n        if best_val_loss > val_loss:\n            best_val_loss = val_loss\n            weights = model.get_weights()\n            \n        print(\"Train acc: {:.4f} | Train Loss: {:.4f} | Validation acc: {:.4f} | Validation Loss: {:.4f}\".format(train_acc, train_loss, val_acc, val_loss))\n    # Load best model\n    model.load_weights(weights)\n    \n    return [train_acc_history, train_loss_history, val_acc_history, val_loss_history], model","metadata":{"execution":{"iopub.status.busy":"2024-04-22T23:07:57.566146Z","iopub.execute_input":"2024-04-22T23:07:57.566493Z","iopub.status.idle":"2024-04-22T23:07:57.583345Z","shell.execute_reply.started":"2024-04-22T23:07:57.566466Z","shell.execute_reply":"2024-04-22T23:07:57.582364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Models","metadata":{}},{"cell_type":"markdown","source":"Inception model","metadata":{}},{"cell_type":"code","source":"inception = models.inception_v3(weights='Inception_V3_Weights.DEFAULT')\n\ninception_model = nn.Sequential(\n    inception.Conv2d_1a_3x3,\n    inception.Conv2d_2a_3x3,\n    inception.Conv2d_2b_3x3,\n    inception.maxpool1,\n    inception.Conv2d_3b_1x1,\n    inception.Conv2d_4a_3x3,\n    inception.maxpool2,\n    inception.Mixed_5b,\n    inception.Mixed_5c,\n    inception.Mixed_5d,\n    inception.Mixed_6a,\n    inception.Mixed_6b,\n    inception.Mixed_6c,\n    inception.Mixed_6d,\n    inception.Mixed_6e,\n    inception.Mixed_7a,\n    inception.Mixed_7b,\n    inception.Mixed_7c,\n    inception.avgpool\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:18.344002Z","iopub.execute_input":"2024-04-23T06:33:18.344672Z","iopub.status.idle":"2024-04-23T06:33:18.692633Z","shell.execute_reply.started":"2024-04-23T06:33:18.34464Z","shell.execute_reply":"2024-04-23T06:33:18.69159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Resnet50 model","metadata":{}},{"cell_type":"code","source":"resnet50 = models.resnet50(weights='ResNet50_Weights.DEFAULT')\n\nresnet50_model = nn.Sequential(\n    resnet50.conv1,\n    resnet50.bn1,\n    resnet50.relu,\n    resnet50.maxpool,\n    resnet50.layer1,\n    resnet50.layer2,\n    resnet50.layer3,\n    resnet50.layer4,\n    resnet50.avgpool\n)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:22.335457Z","iopub.execute_input":"2024-04-23T06:33:22.336328Z","iopub.status.idle":"2024-04-23T06:33:22.83348Z","shell.execute_reply.started":"2024-04-23T06:33:22.336296Z","shell.execute_reply":"2024-04-23T06:33:22.832482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze parameters of pretrained models\nfor param in resnet50_model.parameters():    \n    param.requires_grad = False\n    \nfor param in inception_model.parameters():    \n    param.requires_grad = False","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:23.293805Z","iopub.execute_input":"2024-04-23T06:33:23.294582Z","iopub.status.idle":"2024-04-23T06:33:23.301928Z","shell.execute_reply.started":"2024-04-23T06:33:23.294543Z","shell.execute_reply":"2024-04-23T06:33:23.300834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze training for all \"features\" layers\nfor param_res, param_inc in zip(resnet50.parameters(), inception.parameters()):\n    param_res.requires_grad, param_inc.requires_grad = False, False\n    \n# replace the last fully connected layer with a Linnear layer 133 output\nin_features_resnet50 = resnet50.fc.in_features\nin_features_inception = inception.fc.in_features\n\nresnet50.fc = nn.Linear(in_features_resnet50, 120)\ninception.fc = nn.Linear(in_features_inception, 120)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:26.28117Z","iopub.execute_input":"2024-04-23T06:33:26.282058Z","iopub.status.idle":"2024-04-23T06:33:26.295793Z","shell.execute_reply.started":"2024-04-23T06:33:26.282022Z","shell.execute_reply":"2024-04-23T06:33:26.294761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### model with SGD","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, inception_model, resnet50_model):\n        super(Model,self).__init__()\n        \n        self.inception_model = inception_model\n        self.resnet50_model = resnet50_model        \n        \n        self.output = nn.Sequential(\n            nn.Dropout(0.7),\n            nn.Linear(4096,120)            \n        )\n        \n        self.to(device)\n        # Optimizer \n        self.optim = T.optim.SGD(self.output.parameters(), lr=0.005, momentum=0.9)\n        # Loss\n        self.criterion = T.nn.CrossEntropyLoss()\n        # Scheduler\n        self.scheduler = T.optim.lr_scheduler.StepLR(self.optim, step_size=7, gamma=0.1)\n        \n    def forward(self, x):\n        X1 = self.inception_model(x)\n        X2 = self.resnet50_model(x)\n        \n        X1 = X1.view(X1.size(0), -1)\n        X2 = X2.view(X2.size(0), -1)\n       \n        X = T.cat([X1, X2], dim=1)\n        \n        P = self.output(X)        \n        \n        return P\n    \n    def get_weights(self):\n        return self.output.state_dict()\n    \n    def load_weights(self, weights):\n        self.output.load_state_dict(weights)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T23:44:58.682839Z","iopub.execute_input":"2024-04-22T23:44:58.683229Z","iopub.status.idle":"2024-04-22T23:44:58.693489Z","shell.execute_reply.started":"2024-04-22T23:44:58.683202Z","shell.execute_reply":"2024-04-22T23:44:58.692394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(inception_model, resnet50_model)","metadata":{"execution":{"iopub.status.busy":"2024-04-22T23:45:03.177822Z","iopub.execute_input":"2024-04-22T23:45:03.178442Z","iopub.status.idle":"2024-04-22T23:45:03.199074Z","shell.execute_reply.started":"2024-04-22T23:45:03.17841Z","shell.execute_reply":"2024-04-22T23:45:03.198158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history, model = train_model(train_dl, validation_dl, model)\nT.save(model, 'resnet-inception-sgd.pt')\nT.save(model.state_dict(), 'resnet-inception-sgd-weights.pt')","metadata":{"execution":{"iopub.status.busy":"2024-04-22T23:17:24.318062Z","iopub.execute_input":"2024-04-22T23:17:24.31852Z","iopub.status.idle":"2024-04-22T23:39:56.236224Z","shell.execute_reply.started":"2024-04-22T23:17:24.318483Z","shell.execute_reply":"2024-04-22T23:39:56.234962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model with AdamW","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    \n    def __init__(self, inception_model, resnet50_model):\n        super(Model,self).__init__()\n        \n        self.inception_model = inception_model\n        self.resnet50_model = resnet50_model        \n        \n        self.output = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(4096,120)            \n        )\n        \n        self.to(device)\n        # Optimizer \n        self.optim = T.optim.AdamW(self.output.parameters(), lr=0.005)\n        # Loss\n        self.criterion = T.nn.CrossEntropyLoss()\n        \n    def forward(self, x):\n        X1 = self.inception_model(x)\n        X2 = self.resnet50_model(x)\n        \n        X1 = X1.view(X1.size(0), -1)\n        X2 = X2.view(X2.size(0), -1)\n       \n        X = T.cat([X1, X2], dim=1)\n        \n        P = self.output(X)        \n        \n        return P\n    \n    def get_weights(self):\n        return self.output.state_dict()\n    \n    def load_weights(self, weights):\n        self.output.load_state_dict(weights)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:37.924294Z","iopub.execute_input":"2024-04-23T06:33:37.924969Z","iopub.status.idle":"2024-04-23T06:33:37.933664Z","shell.execute_reply.started":"2024-04-23T06:33:37.924941Z","shell.execute_reply":"2024-04-23T06:33:37.932725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(train_dl, val_dl, model, epochs=50):    \n    \n    train_acc_history = []\n    val_acc_history = []\n    train_loss_history = []\n    val_loss_history = []\n    # Best validation accuracy\n    best_val_loss = 1_000_000.0    \n    # Get initial weights\n    weights = model.get_weights()\n    \n    for epoch in range(epochs):\n        print(\"=\"*20, \"Epoch: \", str(epoch), \"=\"*20)\n        \n        train_correct_pred = 0\n        val_correct_pred = 0\n        train_acc = 0\n        val_acc = 0\n        train_loss = 0\n        val_loss = 0\n        \n        # Set to training mode\n        model.train()\n        \n        for x, y in train_dl:               \n            # Convert data to Tensor            \n            x = x.clone().detach().to(device).requires_grad_(True)\n            y = y.clone().detach().long().to(device)\n            # Reset gradients\n            model.optim.zero_grad()\n            # Predict\n            preds = model(x)            \n            \n            # Compute the loss            \n            loss = model.criterion(preds,y)            \n            \n            # Compute the gradients            \n            loss.backward()\n            # Update weights\n            model.optim.step()\n            # Count the correct predictions\n            preds = T.argmax(preds, dim=1)           \n            train_correct_pred += (preds.long().unsqueeze(1) == y.unsqueeze(1)).sum().item()\n            \n            train_loss += loss.item()           \n        \n        train_acc = train_correct_pred / len(train_dl.dataset)\n        \n        train_acc_history.append(train_acc)\n               \n        train_loss_history.append(train_loss)\n        \n        # Switch to evaluation mode\n        model.eval()        \n        \n        with T.no_grad():\n            for x, y in val_dl:                \n                # Convert data to Tensor                \n                x = x.clone().detach().to(device)\n                y = y.clone().detach().long().to(device)    \n                # Predict\n                preds = model(x)                \n                # Compute the loss\n                loss = model.criterion(preds,y)                                         \n                \n                val_loss += loss.item()                \n                # Count the correct predictions\n                preds = T.argmax(preds, dim=1)\n                \n                val_correct_pred += (preds.long().unsqueeze(1) == y.unsqueeze(1)).sum().item() \n                   \n        \n        val_acc = val_correct_pred / len(val_dl.dataset)\n        \n        val_acc_history.append(val_acc)\n        val_loss_history.append(val_loss)           \n        # Save the weights of the best model\n        if best_val_loss > val_loss:\n            best_val_loss = val_loss\n            weights = model.get_weights()\n            \n        print(\"Train acc: {:.4f} | Train Loss: {:.4f} | Validation acc: {:.4f} | Validation Loss: {:.4f}\".format(train_acc, train_loss, val_acc, val_loss))\n    # Load best model\n    model.load_weights(weights)\n    \n    return [train_acc_history, train_loss_history, val_acc_history, val_loss_history], model","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:48.162196Z","iopub.execute_input":"2024-04-23T06:33:48.162948Z","iopub.status.idle":"2024-04-23T06:33:48.178035Z","shell.execute_reply.started":"2024-04-23T06:33:48.162918Z","shell.execute_reply":"2024-04-23T06:33:48.17707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Model(inception_model, resnet50_model)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:48.693627Z","iopub.execute_input":"2024-04-23T06:33:48.694353Z","iopub.status.idle":"2024-04-23T06:33:48.931754Z","shell.execute_reply.started":"2024-04-23T06:33:48.694322Z","shell.execute_reply":"2024-04-23T06:33:48.930939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history, model = train_model(train_dl, validation_dl, model)\nT.save(model, 'resnet-inception-adamw.pt')\nT.save(model.state_dict(), 'resnet-inception-adamw-weights.pt')","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:33:50.387302Z","iopub.execute_input":"2024-04-23T06:33:50.387932Z","iopub.status.idle":"2024-04-23T06:36:05.349961Z","shell.execute_reply.started":"2024-04-23T06:33:50.387896Z","shell.execute_reply":"2024-04-23T06:36:05.348814Z"},"trusted":true},"execution_count":null,"outputs":[]}]}