{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":325302,"sourceType":"datasetVersion","datasetId":137362},{"sourceId":7520479,"sourceType":"datasetVersion","datasetId":4380888},{"sourceId":7606964,"sourceType":"datasetVersion","datasetId":4429047}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports","metadata":{"_uuid":"f5b41e5a-0d11-449c-a5c9-477b57a65a9a","_cell_guid":"f05ac9cf-11a9-46d6-95a7-baf6d0ac38ee","trusted":true}},{"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport zipfile\nimport glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\n\nimport torchvision\nfrom torchvision import datasets, transforms\nimport torchvision.models as models\nfrom datetime import datetime\nimport wandb\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nimport seaborn as sns\n\nimport copy\n\nrandom_state = 2137\nnp.random.seed(random_state)\ntorch.manual_seed(random_state)\ntorch.cuda.manual_seed(random_state)","metadata":{"_uuid":"dd292565-8513-47b8-a2c6-7a4977473de4","_cell_guid":"6fafdea2-41e8-4cf1-9a6a-f64db210ca65","execution":{"iopub.status.busy":"2024-02-11T21:05:05.710959Z","iopub.execute_input":"2024-02-11T21:05:05.711348Z","iopub.status.idle":"2024-02-11T21:05:08.755816Z","shell.execute_reply.started":"2024-02-11T21:05:05.711314Z","shell.execute_reply":"2024-02-11T21:05:08.754913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define all parameters here! To load network from the checkpoint, define the patch here, and look at the section on the botton of the notebook ","metadata":{}},{"cell_type":"code","source":"passed_config = {\n        'last_checkpoint_path': None,\n        'model': 'ResNet',\n        'optimizer': 'sgd',\n        'learning_rate' : 0.02,\n        'batch_size': 64,\n    }","metadata":{"_uuid":"8b0d9253-f9d3-46a8-bdf7-f7c1bba9e7c7","_cell_guid":"3a882d0b-7f35-4a30-ac61-14d799aee814","execution":{"iopub.status.busy":"2024-02-11T21:05:08.757390Z","iopub.execute_input":"2024-02-11T21:05:08.757746Z","iopub.status.idle":"2024-02-11T21:05:08.762399Z","shell.execute_reply.started":"2024-02-11T21:05:08.757718Z","shell.execute_reply":"2024-02-11T21:05:08.761416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sweep_config = {\n    'method': 'bayes',\n    'name': 'final_tuning_InceptionV3',\n    \"metric\": {\"goal\": \"minimize\", \"name\": \"validation_loss_epoch\"},\n    'parameters': {\n        'model': {'values': ['ResNet101']},\n        'optimizer': {'values': ['sgd', 'adam']},\n        'learning_rate': {'min': 0.0001, 'max': 0.07},\n        'batch_size': {'values': [32, 64]},\n    }\n}","metadata":{"_uuid":"6b812fb1-bee5-48c0-8c6e-16e313ec8c64","_cell_guid":"efa3a7b3-9752-444d-bd1c-c0a9da65a910","execution":{"iopub.status.busy":"2024-02-11T21:05:08.763709Z","iopub.execute_input":"2024-02-11T21:05:08.764083Z","iopub.status.idle":"2024-02-11T21:05:08.773624Z","shell.execute_reply.started":"2024-02-11T21:05:08.764032Z","shell.execute_reply":"2024-02-11T21:05:08.772890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"WANDB_API_KEY\"] = 'put here wandb api key'\nos.environ[\"WANDB_MODE\"] = \"disabled\"","metadata":{"_uuid":"8d04d5bc-4435-4e39-b749-a60ca5a0d63e","_cell_guid":"d720493f-0c28-4daa-b44d-924b21a078fa","execution":{"iopub.status.busy":"2024-02-11T21:05:08.775711Z","iopub.execute_input":"2024-02-11T21:05:08.776005Z","iopub.status.idle":"2024-02-11T21:05:08.783361Z","shell.execute_reply.started":"2024-02-11T21:05:08.775981Z","shell.execute_reply":"2024-02-11T21:05:08.782621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Initialising the project on wandb","metadata":{"_uuid":"54ff73d3-4361-4fb4-aa49-47982d5f86a4","_cell_guid":"49971fda-7849-46de-a925-f87b76c08dfc","trusted":true}},{"cell_type":"code","source":"def initialize_wandb(config = None, config_info = None):\n    current_time = datetime.now().strftime(\"%m-%d_%H:%M\")\n    if config == None:\n        run_name = f\"tuning_{current_time}\"\n        wandb.init()\n    else:\n        run_name = f\"{current_time}_{config_info}\"\n        wandb.init(project='cats&dogs_ML&DL_project', save_code=True, config = config, name = run_name)\n    return","metadata":{"_uuid":"40e08122-5bde-451d-8a5d-1efc92f138a3","_cell_guid":"624d8b13-c4e8-47cd-aaa0-136330fd2c22","execution":{"iopub.status.busy":"2024-02-11T21:05:08.784635Z","iopub.execute_input":"2024-02-11T21:05:08.785414Z","iopub.status.idle":"2024-02-11T21:05:08.792960Z","shell.execute_reply.started":"2024-02-11T21:05:08.785376Z","shell.execute_reply":"2024-02-11T21:05:08.792127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameters","metadata":{"_uuid":"d9b755a5-9bed-4d0c-9379-ada101ebb874","_cell_guid":"60f41da9-366c-4dd9-bff6-9f915e9f23db","trusted":true}},{"cell_type":"code","source":"class configuration:\n    def __init__(self, model, model_transforms, config=None):\n        self.model = model\n        self.initilaized_with_custom_configuration = True\n        if config == None:\n            self.initilaized_with_custom_configuration = False\n            config = wandb.config\n            self.batch_size = config.batch_size\n            self.init_learning_rate = config.learning_rate\n            self.optimizer_name = config.optimizer\n        else:\n            self.last_checkpoint_path = config['last_checkpoint_path']\n            self.init_learning_rate = config['learning_rate']\n            self.optimizer_name = config['optimizer']\n            self.batch_size = config['batch_size']\n        \n        self.optimizer = self.initialize_optimizer()\n        self.scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(self.optimizer, 'min', patience=3, verbose=True, min_lr=self.init_learning_rate/32, factor=0.5)\n        self.criterion = nn.CrossEntropyLoss()\n        self.test_transforms = model_transforms\n        # combine with some basic transforms\n        self.custom_transforms = get_costum_transforms()\n        self.train_transforms = transforms.Compose([self.custom_transforms,model_transforms])\n        \n    def initialize_optimizer(self):\n        if self.optimizer_name == 'sgd':\n            optimizer = optim.SGD(self.model.parameters(), lr=self.init_learning_rate, momentum=0.9)\n        elif self.optimizer_name == 'adam':\n            optimizer = optim.Adam(self.model.parameters(), lr=self.init_learning_rate)\n        else:\n            print(\"Unsupported optimizer:\")\n            raise ValueError(f\"Unsupported optimizer: {optimizer_name}\")\n        return optimizer\n    \n    def get_configuration_info(self):\n        info = f\"{type(self.model).__name__}_\"\n        info += f\"{type(self.optimizer).__name__}_\"\n        info += f\"Batch={self.batch_size}_\"\n        info += f\"lr={self.optimizer.param_groups[0]['lr']}\"\n        return info\n    def get_configuration_dictionary(self):\n        configuration_info = {\n            'model_type': type(self.model).__name__,\n            'criterion': str(self.criterion),\n            'optimizer': type(self.optimizer).__name__,\n            'batch_size': self.batch_size,\n            'scheduler': type(self.scheduler).__name__,\n            'optimizer_params': {\n                'initial_lr': self.optimizer.param_groups[0]['lr']\n            },\n            'scheduler_params': {\n                'min_lr': self.scheduler.min_lrs,\n                'patience': self.scheduler.patience,\n                'factor': self.scheduler.factor\n            },\n            'custom_transforms': [str(transform) for transform in self.custom_transforms.transforms] if self.custom_transforms else None\n        }\n\n        return configuration_info","metadata":{"_uuid":"07422de6-5aab-495c-b98c-a5d8339a27af","_cell_guid":"46ec50f0-1641-4f1b-8c0e-af6d444070cd","_kg_hide-output":false,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-02-11T21:05:08.793979Z","iopub.execute_input":"2024-02-11T21:05:08.794267Z","iopub.status.idle":"2024-02-11T21:05:08.809655Z","shell.execute_reply.started":"2024-02-11T21:05:08.794242Z","shell.execute_reply":"2024-02-11T21:05:08.808747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# loading datasets","metadata":{"_uuid":"3dcf512b-4e5c-413c-aad3-0028f8be4fe0","_cell_guid":"fd3174c1-4aee-4e29-bf1c-2c8b175f5bc8","trusted":true}},{"cell_type":"code","source":"#reading csv\nannotations = pd.read_csv('../input/cats-and-dogs-breeds-classification-oxford-dataset/annotations/annotations/list.txt')\n\n#The first 4 rows consists of the information about breeds\n#Reading the data after 5th row\nannotations = annotations.loc[5:,]\n\n#Processing the columns\nannotations[['CLASS-ID','SPECIES','BREED','ID']] = annotations['#Image CLASS-ID SPECIES BREED ID'].str.split(expand=True) \n\n#Dropping unnecessary columns\nannotations = annotations.drop('#Image CLASS-ID SPECIES BREED ID',axis=1)\n\n#renaming the columns\nannotations = annotations.rename(columns={\"CLASS-ID\": \"image\", \"SPECIES\": \"CLASS-ID\", 'BREED' : \"SPECIES\", \"ID\":\"BREED ID\"})\n\n\n#converting the object type to int type\nannotations[[\"CLASS-ID\",\"SPECIES\",\"BREED ID\"]] = annotations[[\"CLASS-ID\",\"SPECIES\",\"BREED ID\"]].astype(int)","metadata":{"_uuid":"c83b9513-15a6-426a-bcb4-9dd800a72898","_cell_guid":"d1d10af1-daa4-443b-87ab-1694cd1d502d","execution":{"iopub.status.busy":"2024-02-11T21:05:08.810907Z","iopub.execute_input":"2024-02-11T21:05:08.811295Z","iopub.status.idle":"2024-02-11T21:05:08.853416Z","shell.execute_reply.started":"2024-02-11T21:05:08.811260Z","shell.execute_reply":"2024-02-11T21:05:08.852640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RUN THIS CELL ONLY ONCE!!!","metadata":{"_uuid":"5d3184df-d2c8-4cda-8b1b-7784f9c6253e","_cell_guid":"25c82c4e-bf8c-4e06-af65-7b43aaf665c6","trusted":true}},{"cell_type":"code","source":"# adding the extension to image so it can be used to access the real image\nannotations['image'] = annotations['image'].apply(lambda x : str(x)+'.jpg')\nannotations = annotations.reset_index()\nannotations = annotations.drop('index',axis=1)\n\n#Extracting the classname/breed of the animal\nannotations['classname'] = annotations['image'].apply(lambda x: str(x)[:str(x).rindex('_')])\n\n# Adding information about cat or dog based on the 'Species' column to the 'classname' column\nannotations['classname'] = annotations.apply(lambda row: f\"{('dog' if row['SPECIES'] == 2 else 'cat')}_{row['classname']}\", axis=1)","metadata":{"_uuid":"a65a197e-9037-43a1-b283-e715416793a1","_cell_guid":"ddc198d8-0457-4df8-9ae9-1c5b81591540","execution":{"iopub.status.busy":"2024-02-11T21:05:08.854555Z","iopub.execute_input":"2024-02-11T21:05:08.854853Z","iopub.status.idle":"2024-02-11T21:05:08.977420Z","shell.execute_reply.started":"2024-02-11T21:05:08.854827Z","shell.execute_reply":"2024-02-11T21:05:08.976586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Creating dictionary to to link Class-IDs and class names","metadata":{}},{"cell_type":"code","source":"# Drop duplicates to get unique pairs of classID and classname\nclass_mapping_df = annotations[['CLASS-ID', 'classname']].drop_duplicates()\n\n# Create a dictionary to map classID to classname\nclass_id_to_name = dict(zip(class_mapping_df['CLASS-ID'], class_mapping_df['classname']))","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:05:08.978741Z","iopub.execute_input":"2024-02-11T21:05:08.979119Z","iopub.status.idle":"2024-02-11T21:05:08.988122Z","shell.execute_reply.started":"2024-02-11T21:05:08.979079Z","shell.execute_reply":"2024-02-11T21:05:08.987210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Adding transformations","metadata":{"_uuid":"5de72031-2f24-4082-add2-78f5b2997138","_cell_guid":"e4c62191-d037-479d-8003-966d214df6bf","trusted":true}},{"cell_type":"code","source":"def get_costum_transforms():\n    costum_transforms = torchvision.transforms.Compose([\n        transforms.RandomRotation(degrees=(-30, 30),fill=None),\n        transforms.Resize((300,300)),\n        transforms.RandomApply([transforms.Compose([\n                transforms.CenterCrop(200),\n                transforms.RandomCrop(80),\n            ]),], p=0.3),\n        transforms.RandomHorizontalFlip(p=0.4),\n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1),\n        transforms.GaussianBlur(kernel_size=(5,5), sigma=0.3),\n        transforms.RandomApply([transforms.Compose([\n                transforms.GaussianBlur(kernel_size=(9,9), sigma=0.7),\n            ]),], p=0.4),\n      ])\n    return costum_transforms","metadata":{"_uuid":"014a9871-c116-4325-84d9-11d993cc832f","_cell_guid":"1e8ce57e-9537-46bd-93bd-6d48f4d05a30","_kg_hide-output":false,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2024-02-11T21:05:08.991947Z","iopub.execute_input":"2024-02-11T21:05:08.992707Z","iopub.status.idle":"2024-02-11T21:05:09.000119Z","shell.execute_reply.started":"2024-02-11T21:05:08.992677Z","shell.execute_reply":"2024-02-11T21:05:08.999084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n# Creating the dataset class","metadata":{"_uuid":"bc104db6-a1f3-4a07-941f-e6dd9a5683f0","_cell_guid":"01439d6b-2580-4a84-954a-b0e25a369a2a","trusted":true}},{"cell_type":"code","source":"class CatsDogsDataset(Dataset):\n    def __init__(self, file_list, transform=None):\n        self.folder_patch = '/kaggle/input/cats-and-dogs-breeds-classification-oxford-dataset/images/images/'\n        self.annotations = file_list\n        self.transform = transform\n        self.filelength = len(file_list)\n\n    def __len__(self):\n        return self.filelength\n\n    def __getitem__(self, idx):\n        classID = self.annotations['CLASS-ID'].iloc[idx]\n        img_path = self.annotations['image'].iloc[idx]\n        img_path = self.folder_patch + img_path\n        img = Image.open(img_path)\n        \n        has_alpha_channel = img.mode == 'RGBA'\n        if has_alpha_channel == True:\n            #print(\"image has Alpha channel\")\n            img = img.convert('RGB')\n        if self.transform is not None:\n            try:\n                img = self.transform(img)\n            except RuntimeError as e:\n                print(f\"Exception: {e}\")\n                print(\"Shape before normalization:\", img.size)\n                print(img_path)\n                tot = transforms.ToTensor()\n                img_tensor = tot(img)\n                print(\"Input Tensor Shape:\", img_tensor.shape)\n                print(\"Input Tensor Values:\", img_tensor)\n        return img, classID-1","metadata":{"_uuid":"678f73e2-736c-4b98-a6c4-d0084ff8955d","_cell_guid":"dc8cbeab-2d61-4464-be7e-ea5998ea135e","execution":{"iopub.status.busy":"2024-02-11T21:05:09.002728Z","iopub.execute_input":"2024-02-11T21:05:09.003101Z","iopub.status.idle":"2024-02-11T21:05:09.013136Z","shell.execute_reply.started":"2024-02-11T21:05:09.003064Z","shell.execute_reply":"2024-02-11T21:05:09.012170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_images = CatsDogsDataset(annotations)","metadata":{"_uuid":"ca49b3cf-0779-4055-91f0-94e64feee06c","_cell_guid":"807c483d-7f9d-4b67-8537-178e9c284178","execution":{"iopub.status.busy":"2024-02-11T21:05:09.014257Z","iopub.execute_input":"2024-02-11T21:05:09.014576Z","iopub.status.idle":"2024-02-11T21:05:09.024909Z","shell.execute_reply.started":"2024-02-11T21:05:09.014548Z","shell.execute_reply":"2024-02-11T21:05:09.024153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting datasets and creating the dataloaders","metadata":{"_uuid":"9efe1d59-3943-4ea5-ab01-0d8cfea4ca3d","_cell_guid":"b6fed058-bdf1-4cb9-a666-6801541fb8c3","trusted":true}},{"cell_type":"code","source":"def get_dataloaders(annotations, config): \n    train_set_temp, test_annotations = train_test_split(annotations, test_size=0.2, random_state=random_state, stratify=annotations['CLASS-ID'])\n    train_annotations, validation_annotations = train_test_split(train_set_temp, test_size=0.2, random_state=random_state, stratify=train_set_temp['CLASS-ID'])\n\n    train_data = CatsDogsDataset(train_annotations, transform=config.train_transforms)\n    valid_data = CatsDogsDataset(validation_annotations, transform=config.test_transforms)\n    test_data = CatsDogsDataset(test_annotations, transform=config.test_transforms)\n\n    train_loader = DataLoader(dataset=train_data, batch_size=config.batch_size , shuffle=True)\n    valid_loader = DataLoader(dataset=valid_data, batch_size=config.batch_size, shuffle=True)\n    test_loader = DataLoader(dataset=test_data, batch_size=config.batch_size, shuffle=False)\n    \n    return train_loader,  valid_loader, test_loader","metadata":{"_uuid":"7b09690e-55d9-45a4-a758-cd93632a811b","_cell_guid":"3d204d7d-4140-4554-8394-da7d27835371","execution":{"iopub.status.busy":"2024-02-11T21:05:09.025994Z","iopub.execute_input":"2024-02-11T21:05:09.026328Z","iopub.status.idle":"2024-02-11T21:05:09.034845Z","shell.execute_reply.started":"2024-02-11T21:05:09.026291Z","shell.execute_reply":"2024-02-11T21:05:09.033894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{"_uuid":"2e2488d5-d2a4-4c4c-bfee-70c6b10acb68","_cell_guid":"ffea4198-5702-4889-bc8b-6e84246b230d","trusted":true}},{"cell_type":"markdown","source":"### Accuracy function","metadata":{"_uuid":"2bed54b5-5fff-40c6-8ca2-b2a4594349cb","_cell_guid":"9409ae66-18bc-4297-9278-bb4081319ef6","trusted":true}},{"cell_type":"code","source":"def my_accuracy(predictions, labels):\n    predictions = torch.argmax(predictions,dim=1)\n    correct = (predictions == labels)\n    acc = sum(correct) / len(predictions)\n    return acc.item()","metadata":{"_uuid":"c6faa626-6dda-4d5e-8c39-a63c93340f40","_cell_guid":"ff039d46-8087-4803-964b-c2f5c8a8f8c9","execution":{"iopub.status.busy":"2024-02-11T21:05:09.035984Z","iopub.execute_input":"2024-02-11T21:05:09.036262Z","iopub.status.idle":"2024-02-11T21:05:09.047734Z","shell.execute_reply.started":"2024-02-11T21:05:09.036237Z","shell.execute_reply":"2024-02-11T21:05:09.046773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving checkpoint","metadata":{"_uuid":"a4637c80-ac5c-489e-b4a1-156631578603","_cell_guid":"b97b9e8a-1172-4bdb-b5f3-0f9d3a56ceb4","trusted":true}},{"cell_type":"code","source":"def save_checkpoint(initilaized_with_custom_configuration, epoch, model, optimizer, loss, config):\n    if initilaized_with_custom_configuration == True:\n        if epoch%1 == 0:\n            checkpoint = {\n                'epoch': epoch,\n                'model_state_dict': model.state_dict(prefix='modified_fc.'),\n                'optimizer_state_dict': optimizer.state_dict(), # contains information like lr scheduler state\n                'loss': loss,  # Save the current training loss if needed\n            }\n            info = config.get_configuration_info()\n            checkpoint_path = f'/kaggle/working/{epoch}_{info}.pth'\n            torch.save(checkpoint, checkpoint_path)\n            config.last_checkpoint_path = checkpoint_path\n    return","metadata":{"_uuid":"b80a8abc-5349-47d7-83bb-a0c5a1b1120c","_cell_guid":"63936c3b-4f4c-43d0-af07-8faaf7c8d58c","execution":{"iopub.status.busy":"2024-02-11T21:05:09.048869Z","iopub.execute_input":"2024-02-11T21:05:09.049149Z","iopub.status.idle":"2024-02-11T21:05:09.056307Z","shell.execute_reply.started":"2024-02-11T21:05:09.049124Z","shell.execute_reply":"2024-02-11T21:05:09.055558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training loop","metadata":{"_uuid":"967732e8-2b82-4cbc-91b4-b70863ddbf08","_cell_guid":"6cc568ce-e3b4-4c36-a4bc-aeb7625ec7d4","trusted":true}},{"cell_type":"code","source":"class TrainingManager:\n    _instance = None\n    \n    def __new__(cls, *args, **kwargs):\n        if not cls._instance:\n            print('inintializing manager for the first time')\n            cls._instance = super().__new__(cls)\n            cls._instance.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n            cls._instance.config = None\n            \n            cls._instance.train_loader = None\n            cls._instance.valid_loader = None\n            cls._instance.test_loader = None\n            cls._instance.resume_epoch = None\n            \n            cls._instance.training_accuracy_epoch = 0\n            cls._instance.validation_accuracy_epoch = 0\n            cls._instance.training_loss_epoch = 0\n            cls._instance.validation_loss_epoch = 100\n            cls._instance.total_train_batches = 0\n            cls._instance.total_valid_batches = 0\n            cls._instance.resume_epoch = 0\n            cls._instance.cnt = 0\n            cls._instance.log_interval = 1\n            cls._instance.mean_loss = 0.0\n            cls._instance.mean_lr = 0.0\n            cls._instance.mean_grad_magnitude = 0.0\n            cls._instance.termination_counter = 0\n            cls._instance.last_validation_loss = 0\n        return cls._instance\n\n    def setup_configuration(self, config, annotations):\n        self.config = config\n        self.config.model.to(self.device)\n        self.resume_epoch = 0\n        self.train_loader, self.valid_loader, self.test_loader = get_dataloaders(annotations, config)\n        \n        self.total_train_batches = len(self.train_loader)\n        self.total_valid_batches = len(self.valid_loader)\n        \n        if self.config.initilaized_with_custom_configuration == True:\n            if self.config.last_checkpoint_path != None:\n                checkpoint = torch.load(self.config.last_checkpoint_path)\n                self.config.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n                self.resume_epoch = checkpoint['epoch']\n                # Remove the prefix from the keys\n                state_dict = {k.replace('modified_fc.', ''): v for k, v in checkpoint['model_state_dict'].items()}\n                # It is required after DataParallel wraps the module\n                state_dict = {k.replace('module.', ''): v for k, v in state_dict.items()}\n                self.config.model.load_state_dict(state_dict)\n            else:\n                print('no checkpoint found')\n        d = next(self.config.model.parameters()).device\n        if d.type == 'cuda':\n            print(\"Model is on GPU\")\n            self.config.model = torch.nn.DataParallel(self.config.model) # if multiple GPUs use them\n        else:\n            print(\"Model is on CPU\")\n                \n    def training(self, config, annotations):\n        num_epochs = 40\n        self.termination_counter = 0\n        self.setup_configuration(config, annotations)\n        self.config.model.train()\n        for epoch in range(num_epochs - self.resume_epoch):\n            self.training_loss_epoch = 0\n            self.training_accuracy_epoch = 0\n            self.cnt = 0\n\n            for train_features_batch, train_labels_batch in self.train_loader:\n                train_features_batch, train_labels_batch = train_features_batch.to(self.device), train_labels_batch.to(self.device)\n                self.config.optimizer.zero_grad()\n\n                y_hat = self.config.model(train_features_batch)\n                \n                training_loss = self.config.criterion(y_hat, train_labels_batch)\n                training_loss.backward()\n                self.config.optimizer.step()\n\n                self.training_loss_epoch += training_loss.item()\n                train_accuracy = my_accuracy(y_hat, train_labels_batch)\n                self.training_accuracy_epoch += train_accuracy\n                \n                self.cnt += 1\n                self.batch_log_metrics(training_loss)\n                \n            save_checkpoint(self.config.initilaized_with_custom_configuration,epoch+self.resume_epoch, self.config.model, self.config.optimizer, self.training_loss_epoch, config)\n            self.last_validation_loss = self.validation_loss_epoch\n            self.validation_loss_epoch = 0\n            self.validation_accuracy_epoch = 0\n            self.cnt = 0\n\n            for val_features_batch, val_labels_batch in self.valid_loader:\n                val_features_batch, val_labels_batch = val_features_batch.to(self.device), val_labels_batch.to(self.device)\n                with torch.no_grad():\n                    y_hat_val = self.config.model(val_features_batch)\n                    validation_loss = self.config.criterion(y_hat_val, val_labels_batch)\n                    self.validation_loss_epoch += validation_loss.item()\n                    val_accuracy = my_accuracy(y_hat_val, val_labels_batch)\n                    self.validation_accuracy_epoch += val_accuracy\n                    self.cnt += 1\n                    \n            self.normalize_metrics()      \n            self.log_metrics()\n            self.print_metrics(epoch + self.resume_epoch)\n            self.config.scheduler.step(self.validation_loss_epoch)\n            # Early stopping\n            if  (self.validation_accuracy_epoch - self.training_accuracy_epoch < -0.20) and (self.training_loss_epoch - self.validation_loss_epoch < -0.15):\n                save_checkpoint(self.config.initilaized_with_custom_configuration, epoch+self.resume_epoch, self.config.model, self.config.optimizer, self.training_loss_epoch, config)\n                print('Early stopping! Overfitting')\n                return\n            if  self.validation_loss_epoch >= self.last_validation_loss:\n                self.termination_counter += 1\n                save_checkpoint(self.config.initilaized_with_custom_configuration, epoch+self.resume_epoch, self.config.model, self.config.optimizer, self.training_loss_epoch, config)\n                if self.termination_counter > 3:\n                    print('Early stopping! Overfitting')\n                    return\n            else:\n                self.termination_counter = 0\n        save_checkpoint(self.config.initilaized_with_custom_configuration, epoch+self.resume_epoch, self.config.model, self.config.optimizer, self.training_loss_epoch, config)\n        return\n    def normalize_metrics(self):\n            self.training_accuracy_epoch = self.training_accuracy_epoch / self.total_train_batches\n            self.validation_accuracy_epoch = self.validation_accuracy_epoch / self.total_valid_batches\n            self.training_loss_epoch = self.training_loss_epoch / self.total_train_batches\n            self.validation_loss_epoch = self.validation_loss_epoch / self.total_valid_batches\n            \n    def log_metrics(self):\n        wandb.log({f'validation_loss_epoch': self.validation_loss_epoch,\n                   f'training_loss_epoch': self.training_loss_epoch})\n        wandb.log({f'training_accuracy_epoch': self.training_accuracy_epoch,\n                   f'validation_accuracy_epoch': self.validation_accuracy_epoch})\n\n    def print_metrics(self, epoch):\n        print('~~~~~~~~~~~~~~~~~~~~~ Epoch: ', epoch, ' ~~~~~~~~~~~~~~~~~~~~~')\n        print({f'training_loss_epoch': self.training_loss_epoch,\n               f'training_accuracy_epoch': self.training_accuracy_epoch})\n        print({f'validation_loss_epoch': self.validation_loss_epoch,\n               f'validation_accuracy_epoch': self.validation_accuracy_epoch})\n        \n    def batch_log_metrics(self, train_loss):\n        # Calculate mean values every n batches\n        self.mean_loss += train_loss.item()\n        self.mean_lr += self.config.optimizer.param_groups[0]['lr']\n        self.mean_grad_magnitude += self.calculate_gradient_magnitude()\n\n        if self.cnt % self.log_interval == 0:\n            # Log mean values\n            mean_loss_batch = self.mean_loss / self.log_interval\n            mean_lr_batch = self.mean_lr / self.log_interval\n            mean_grad_magnitude_batch = self.mean_grad_magnitude / self.log_interval\n\n            wandb.log({'training_loss_every_n_batches': mean_loss_batch,\n                       'training_learning_rate_every_n_batches': mean_lr_batch,\n                       'training_gradient_magnitude_every_n_batches': mean_grad_magnitude_batch})\n\n            # Reset mean values\n            self.mean_loss = 0.0\n            self.mean_lr = 0.0\n            self.mean_grad_magnitude = 0.0\n            \n    def calculate_gradient_magnitude(self):\n        # Example implementation of calculating the gradient magnitude\n        total_norm = 0.0\n        for param in self.config.model.parameters():\n            if param.grad is not None:\n                total_norm += param.grad.data.norm(2).item()\n        return total_norm\n        \n    def evaluate_on_test_dataset(self):\n        device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n        self.config.model.eval()\n        predictions = []\n        labels = []\n        with torch.no_grad():\n            for inputs_batch, labels_batch in self.test_loader:\n                inputs_batch = inputs_batch.to(self.device)\n                labels_batch = labels_batch.tolist()\n                outputs = self.config.model(inputs_batch)\n                predicted_classes = [torch.argmax(pred).item() for pred in outputs]\n                predictions += predicted_classes\n                labels += labels_batch\n        return accuracy_score(labels, predictions), confusion_matrix(labels, predictions)","metadata":{"_uuid":"420cc670-9d12-4fdf-baf3-a1fc4ba8ce7e","_cell_guid":"75fd7445-fb6e-4112-885c-19f166ad754c","execution":{"iopub.status.busy":"2024-02-11T21:05:09.057520Z","iopub.execute_input":"2024-02-11T21:05:09.057800Z","iopub.status.idle":"2024-02-11T21:05:09.093261Z","shell.execute_reply.started":"2024-02-11T21:05:09.057777Z","shell.execute_reply":"2024-02-11T21:05:09.092040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining models","metadata":{"_uuid":"63e21653-39e1-42f2-9c24-d447dfd03a61","_cell_guid":"6c40ad0a-e194-4d86-a354-d34ddacf9809","trusted":true}},{"cell_type":"markdown","source":"### ResNet101","metadata":{"_uuid":"95c2a756-ddf1-4cfc-a9a7-a1fb9869d7a7","_cell_guid":"e515a9ae-44ec-4e14-b71a-e7fa4e9529c1","trusted":true}},{"cell_type":"code","source":"def get_pretrained_resnet101():\n    model = models.resnet101(weights=torchvision.models.ResNet101_Weights.DEFAULT)\n    num_classes = 37\n    model.fc = nn.Sequential(\n        nn.Linear(model.fc.in_features, num_classes),\n        nn.Softmax(dim=1)\n    )\n    nn.init.xavier_normal_(model.fc[0].weight)\n    nn.init.constant_(model.fc[0].bias, 0)\n    \n    weights = torchvision.models.ResNet101_Weights.DEFAULT\n    preprocess_transforms = weights.transforms()\n    \n    return model, preprocess_transforms","metadata":{"_uuid":"adcfbbc2-844a-4ae1-b6f1-0cf364f48dbc","_cell_guid":"81d61e69-651c-40f6-9241-dc38acb40934","execution":{"iopub.status.busy":"2024-02-11T21:05:09.094652Z","iopub.execute_input":"2024-02-11T21:05:09.095026Z","iopub.status.idle":"2024-02-11T21:05:09.106519Z","shell.execute_reply.started":"2024-02-11T21:05:09.094991Z","shell.execute_reply":"2024-02-11T21:05:09.105646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Inception V3","metadata":{}},{"cell_type":"code","source":"def remove_auxiliary_logits(model):\n    if isinstance(model, models.Inception3):\n        # Set the auxiliary logits to False\n        model.aux_logits = False\n        \n        # Replace the auxiliary classifier with an identity layer\n        model.AuxLogits = nn.Identity()\n\ndef get_pretrained_inception_v3():\n    model = torchvision.models.inception_v3(pretrained=True)\n    \n    remove_auxiliary_logits(model)\n    \n    # Replace the final fully connected layer with a new one\n    num_classes = 37\n    model.fc = nn.Sequential(\n        nn.Linear(model.fc.in_features, num_classes),\n        nn.Softmax(dim=1)\n    )\n    \n    # Initialize the new fully connected layer\n    nn.init.xavier_normal_(model.fc[0].weight)\n    nn.init.constant_(model.fc[0].bias, 0)\n    \n    # Get the default preprocessing transforms for Inception V3\n    preprocess_transforms = torchvision.transforms.Compose([\n        torchvision.transforms.Resize((299,299)),\n        torchvision.transforms.ToTensor(),\n        torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                                         std=[0.229, 0.224, 0.225])\n    ])\n    \n    return model, preprocess_transforms","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:05:09.107590Z","iopub.execute_input":"2024-02-11T21:05:09.107899Z","iopub.status.idle":"2024-02-11T21:05:09.117405Z","shell.execute_reply.started":"2024-02-11T21:05:09.107874Z","shell.execute_reply":"2024-02-11T21:05:09.116591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Functions to call training for training and sweep optimization","metadata":{}},{"cell_type":"code","source":"def train_model(config):\n    if config['model'] == 'ResNet':\n        model, preprocess_transforms = get_pretrained_resnet101()\n    elif config['model'] == 'Inception':\n        model, preprocess_transforms = get_pretrained_inception_v3()\n    else:\n        print('wrong model declaration')\n        return\n    traning_config = configuration(model, preprocess_transforms, config)\n    initialize_wandb(config, traning_config.get_configuration_info())\n    manager = TrainingManager()\n    manager.training(traning_config, annotations)\n    return","metadata":{"_uuid":"74d527f8-6efe-4090-9a1d-54280263d7af","_cell_guid":"6f01a231-f65c-4774-a861-4903d57ced0b","execution":{"iopub.status.busy":"2024-02-11T21:05:09.118512Z","iopub.execute_input":"2024-02-11T21:05:09.118923Z","iopub.status.idle":"2024-02-11T21:05:09.131028Z","shell.execute_reply.started":"2024-02-11T21:05:09.118897Z","shell.execute_reply":"2024-02-11T21:05:09.130078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### SPECIFY MODEL FOR SWEEP OPTIMISATION HERE!!!","metadata":{}},{"cell_type":"code","source":"def start_training_with_sweep():\n    wandb.init()\n    print(wandb.config)\n    print(\"*\")\n    model, preprocess_transforms = get_pretrained_inception_v3()\n    traning_config = configuration(model, preprocess_transforms)\n    manager = TrainingManager()\n    manager.training(traning_config, annotations)\n    return","metadata":{"_uuid":"7bd1f441-f280-403d-a46c-31cfe9985e8d","_cell_guid":"d1b937ea-843a-43f2-ab3b-71baaa83734b","execution":{"iopub.status.busy":"2024-02-11T21:05:09.132100Z","iopub.execute_input":"2024-02-11T21:05:09.132430Z","iopub.status.idle":"2024-02-11T21:05:09.141336Z","shell.execute_reply.started":"2024-02-11T21:05:09.132404Z","shell.execute_reply":"2024-02-11T21:05:09.140485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load and evaluate the model","metadata":{}},{"cell_type":"code","source":"def evaluate_model():\n    manager = TrainingManager()\n    accuracy, cm = manager.evaluate_on_test_dataset()\n    \n    print('prediction accuracy = ', accuracy)\n    \n    plt.figure(figsize=(8, 8))\n    sns.heatmap(cm, annot=True, cmap='Blues', fmt='g', cbar=True)\n    plt.xlabel('Predicted')\n    plt.ylabel('True')\n    plt.title('Confusion Matrix')\n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:05:09.142509Z","iopub.execute_input":"2024-02-11T21:05:09.142780Z","iopub.status.idle":"2024-02-11T21:05:09.150852Z","shell.execute_reply.started":"2024-02-11T21:05:09.142751Z","shell.execute_reply":"2024-02-11T21:05:09.149974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checkpoint patch should be specified in config\ndef load_model_from_checkpoint(config):\n    if config['model'] == 'ResNet':\n        model, preprocess_transforms = get_pretrained_resnet101()\n    elif config['model'] == 'Inception':\n        model, preprocess_transforms = get_pretrained_inception_v3()\n    else:\n        print('wrong model declaration')\n        return\n    testing_config = configuration(model, preprocess_transforms, config)\n    testing_config.last_checkpoint_path = config['last_checkpoint_path']\n    manager = TrainingManager()\n    manager.setup_configuration(testing_config, annotations)\n    manager = TrainingManager()\n    return","metadata":{"_uuid":"60502f16-c14c-42cb-85bf-e547a99db699","_cell_guid":"27e44661-9807-47be-9a0b-050a849cae6d","execution":{"iopub.status.busy":"2024-02-11T21:05:09.151990Z","iopub.execute_input":"2024-02-11T21:05:09.152762Z","iopub.status.idle":"2024-02-11T21:05:09.162977Z","shell.execute_reply.started":"2024-02-11T21:05:09.152727Z","shell.execute_reply":"2024-02-11T21:05:09.162127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calling functionalities","metadata":{}},{"cell_type":"code","source":"train_model(passed_config)","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:05:09.164000Z","iopub.execute_input":"2024-02-11T21:05:09.164333Z","iopub.status.idle":"2024-02-11T21:10:47.922236Z","shell.execute_reply.started":"2024-02-11T21:05:09.164307Z","shell.execute_reply":"2024-02-11T21:10:47.920960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"evaluate_model()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:19:05.007930Z","iopub.execute_input":"2024-02-11T21:19:05.008350Z","iopub.status.idle":"2024-02-11T21:19:22.541464Z","shell.execute_reply.started":"2024-02-11T21:19:05.008317Z","shell.execute_reply":"2024-02-11T21:19:22.540491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load_model_from_checkpoint(passed_config)\n#evaluate_model()","metadata":{"execution":{"iopub.status.busy":"2024-02-11T21:16:33.013865Z","iopub.execute_input":"2024-02-11T21:16:33.014298Z","iopub.status.idle":"2024-02-11T21:16:50.572755Z","shell.execute_reply.started":"2024-02-11T21:16:33.014264Z","shell.execute_reply":"2024-02-11T21:16:50.571922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb_mode = os.environ.get(\"WANDB_MODE\")\nif wandb_mode != 'disabled':\n    initialize_wandb()\n    sweep_id = wandb.sweep(sweep=sweep_config, project=\"cats&dogs_ML&DL_project\")\n    #print(wandb.config)\n    wandb.agent(sweep_id, function=start_training_with_sweep, count=14)\nelse:\n    print('wandb is disabled. To run sweeps enable wandb and pass teh right api key')","metadata":{"_uuid":"db4dba4f-326b-41b5-8732-b8199756c4b5","_cell_guid":"a3848d77-c012-46cf-9799-cb80a56d8dd2","execution":{"iopub.status.busy":"2024-02-11T21:10:49.810660Z","iopub.status.idle":"2024-02-11T21:10:49.811101Z","shell.execute_reply.started":"2024-02-11T21:10:49.810877Z","shell.execute_reply":"2024-02-11T21:10:49.810900Z"},"trusted":true},"execution_count":null,"outputs":[]}]}