{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":7327,"databundleVersionId":861871,"sourceType":"competition"}],"dockerImageVersionId":30699,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport torch\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import Dataset\nimport torch.optim as optim\nimport torchvision as tv\nfrom torchvision import models\nfrom torchvision.models import resnet50, inception_v3\nfrom skimage.io import imread\nimport numpy as np\nfrom sklearn.metrics import f1_score, multilabel_confusion_matrix, classification_report, accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nimport matplotlib.pyplot as plt\ntry:\n    import onnx\nexcept:\n    %pip install onnx\n    import onnx","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-06T16:10:11.862195Z","iopub.execute_input":"2024-05-06T16:10:11.862952Z","iopub.status.idle":"2024-05-06T16:10:11.871087Z","shell.execute_reply.started":"2024-05-06T16:10:11.862921Z","shell.execute_reply":"2024-05-06T16:10:11.870157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Data CSV","metadata":{}},{"cell_type":"code","source":"# Define the root directory\nroot_dir = '/kaggle/input/dog-breed-identification/'\n\n# Read data labels\ndf = pd.read_csv(root_dir+'labels.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:11.872523Z","iopub.execute_input":"2024-05-06T16:10:11.872785Z","iopub.status.idle":"2024-05-06T16:10:11.905612Z","shell.execute_reply.started":"2024-05-06T16:10:11.872764Z","shell.execute_reply":"2024-05-06T16:10:11.904716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()\nprint()\nprint(df[\"breed\"].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:11.907402Z","iopub.execute_input":"2024-05-06T16:10:11.908126Z","iopub.status.idle":"2024-05-06T16:10:11.921642Z","shell.execute_reply.started":"2024-05-06T16:10:11.908095Z","shell.execute_reply":"2024-05-06T16:10:11.920704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split & One-hot-encode","metadata":{}},{"cell_type":"code","source":"# Load the data from the DataFrame and perform a train-test-split\ntrain, val = train_test_split(df, test_size = 0.2, stratify = df[\"breed\"])\n\n# One-hot encode the labels\ntrain_encoded = pd.get_dummies(train['breed'])\nval_encoded = pd.get_dummies(val['breed'])\n\n# Concatenate the one-hot encoded labels DataFrame with the original DataFrame\ntrain_one_hot = train[['id']].join(train_encoded)\nval_one_hot = val[['id']].join(val_encoded)\n\n# Display the modified DataFrame\ntrain_one_hot.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:11.922692Z","iopub.execute_input":"2024-05-06T16:10:11.922956Z","iopub.status.idle":"2024-05-06T16:10:11.974209Z","shell.execute_reply.started":"2024-05-06T16:10:11.922925Z","shell.execute_reply":"2024-05-06T16:10:11.973369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CustomDataset Class\n\nThe `CustomDataset` class defines a custom dataset for training a neural network model with data transformations (augmentations). The dataset is designed to load images from a DataFrame containing file paths and labels of images.","metadata":{}},{"cell_type":"code","source":"# ImageNet dist\ntrain_mean = [0.485, 0.456, 0.406]\ntrain_std = [0.229, 0.224, 0.225]\n\n\nclass CustomDataset(Dataset):\n    def __init__(self, data, mode, path):\n        self.data = data  # Df containing filepaths and labels of imgs\n        self.mode = mode  # training or test\n        self.path_prefix = path\n\n        # Data transformations\n        self._transform_t = tv.transforms.Compose([\n            tv.transforms.ToPILImage(),\n            \n            # tv.transforms.RandomCrop((300, 300), 30, True, padding_mode='reflect'),\n            # tv.transforms.ToPILImage(),\n            tv.transforms.Resize((299,299)),\n#             tv.transforms.Resize((325,325)),\n#             tv.transforms.CenterCrop(size = 299),\n            tv.transforms.RandomHorizontalFlip(0.5),\n#             tv.transforms.RandomVerticalFlip(0.5),\n            tv.transforms.RandomRotation(degrees = 50),\n            tv.transforms.ColorJitter(),\n            tv.transforms.ToTensor(),\n            tv.transforms.Normalize(train_mean, train_std),\n        ])\n        self._transform_v = tv.transforms.Compose([\n            tv.transforms.ToPILImage(),\n            tv.transforms.Resize((299,299)),\n#             tv.transforms.CenterCrop(size = 299),\n            tv.transforms.ToTensor(),\n            tv.transforms.Normalize(train_mean, train_std)\n        ])\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, index):\n        img_path = os.path.join(self.path_prefix + self.data.iloc[index][\"id\"]+'.jpg')\n        channel_last = imread(img_path)\n        # img = np.transpose(channel_last, (2, 0, 1))\n\n        # data transformations\n        if self.mode == 'training':\n            transformed = self._transform_t(channel_last)\n        else:\n            transformed = self._transform_v(channel_last)\n        \n        if self.mode != 'test':\n            # load labels\n            label = torch.tensor(self.data.iloc[index, 1:], dtype=torch.float32)\n            return transformed, label\n        else:\n            return transformed","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:34:16.287265Z","iopub.execute_input":"2024-05-06T16:34:16.287994Z","iopub.status.idle":"2024-05-06T16:34:16.298895Z","shell.execute_reply.started":"2024-05-06T16:34:16.287964Z","shell.execute_reply":"2024-05-06T16:34:16.297952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Trainer Class\n\nThe `Trainer` class is designed to facilitate training neural network models. Below is a summary of its functionalities:\n\n## Initialization:\n- `model`: The neural network model to be trained.\n- `crit`: The loss function used for optimization.\n- `optim`: The optimizer for updating model parameters.\n- `train_dl`: Training dataset loader.\n- `val_test_dl`: Validation or test dataset loader.\n- `cuda`: Whether to use GPU for computation.\n- `early_stopping_patience`: The number of epochs with no improvement after which training will be stopped.\n\n## Methods:\n1. `save_checkpoint`: Saves the current model state as a checkpoint.\n2. `restore_best_checkpoint`: Restores the model weights from the epoch with the lowest validation loss.\n3. `save_onnx`: Saves the model in ONNX format.\n4. `train_step`: Performs a single training step (forward pass, backward pass, and parameter update).\n5. `val_test_step`: Performs a single validation/test step (forward pass to compute loss).\n6. `train_epoch`: Trains the model for one epoch and calculates training loss and accuracy.\n7. `val_test`: Evaluates the model on the validation/test set and calculates loss and accuracy.\n8. `metric_calc`: Calculates accuracy based on predicted labels and true labels.\n9. `fit`: Trains the model for multiple epochs, optionally implementing early stopping.\n10. `lr_scheduler`: Sets up a learning rate scheduler for the optimizer.\n\n","metadata":{}},{"cell_type":"code","source":"class Trainer:\n\n    def __init__(self,\n                 model,  # Model to be trained.\n                 crit,  # Loss function\n                 optim=None,  # Optimizer\n                 train_dl=None,  # Training data set\n                 val_test_dl=None,  # Validation (or test) data set\n                 cuda=True,  # Whether to use the GPU\n                 early_stopping_patience=-1):  # The patience for early stopping\n        self._model = model\n        self._crit = crit\n        self._optim = optim\n        self._train_dl = train_dl\n        self._val_test_dl = val_test_dl\n        self._cuda = cuda\n        self._early_stopping_patience = early_stopping_patience\n        self.checkpoint = None\n        self.scheduler = None\n\n        if cuda:\n            self._model = model.cuda()\n            self._crit = crit.cuda()\n\n    def save_checkpoint(self):\n        '''Overwriting checkpoint whenever validation loss is improved'''\n        self.checkpoint = self._model.state_dict()\n\n    def restore_best_checkpoint(self):\n        '''Restore model weights of epoch with best validation loss'''\n        self._model.load_state_dict(self.checkpoint)\n\n    def save_onnx(self, fn):\n        '''Save model as .onnx-file'''\n        folder_name = 'checkpoints'\n        if not os.path.exists(folder_name):\n            os.makedirs(folder_name)\n        m = self._model.cpu()\n        m.eval()\n        x = torch.randn(1, 3, 300, 300, requires_grad=True)\n        y = self._model(x)\n        torch.onnx.export(m,  # model being run\n                          x,  # model input (or a tuple for multiple inputs)\n                          fn,  # where to save the model (can be a file or file-like object)\n                          export_params=True,  # store the trained parameter weights inside the model file\n                          opset_version=10,  # the ONNX version to export the model to\n                          do_constant_folding=True,  # whether to execute constant folding for optimization\n                          input_names=['input'],  # the model's input names\n                          output_names=['output'],  # the model's output names\n                          dynamic_axes={'input': {0: 'batch_size'},  # variable lenght axes\n                                        'output': {0: 'batch_size'}})\n\n    def train_step(self, x, y):\n\n        # Reset gradients to zero\n        self._optim.zero_grad()\n\n        # Forward pass: compute predicted y\n        y_hat = self._model(x)\n        y_hat = y_hat[0] # inception-model\n\n        # Compute loss\n        loss = self._crit(y_hat, y)\n\n        # Backward pass: compute gradient of the loss with respect to model parameters\n        loss.backward()\n\n        # Update model parameters\n        self._optim.step()\n\n        # Return loss and predicted y\n        return loss, y_hat\n\n    def val_test_step(self, x, y):\n\n        # Forward pass: compute predicted y\n        y_hat = self._model(x)\n\n        # Compute validation loss\n        loss = self._crit(y_hat, y)\n\n        # Return validation loss and predicted y\n        return loss, y_hat\n\n    def train_epoch(self):\n\n        # Set training mode\n        self._model.train()\n        total_loss = 0\n        predictions = []\n        targets = []\n\n        # Iterate over batches in the training set\n        for i, batch in enumerate(self._train_dl):\n            imgs, labels = batch\n\n            # Move tensors to GPU if available\n            if self._cuda:\n                imgs = imgs.cuda()\n                labels = labels.cuda()\n\n            loss, preds = self.train_step(imgs, labels)\n            targets.extend(labels.cpu().detach().numpy())\n            predictions.extend(preds.cpu().detach().numpy())\n            total_loss += loss.item()\n\n        # Calculate average batch loss\n        avg_loss = total_loss / len(self._train_dl)\n\n        # Calculate custom metric\n        metric = self.metric_calc(np.array(targets), np.array(predictions))\n        print(f\"\\tTraining Loss: {avg_loss}, Training Accuracy: {metric}\")\n\n        return avg_loss\n\n    def val_test(self):\n\n        # Set evaluation mode\n        self._model.eval()\n        total_loss = 0\n        predictions = []\n        targets = []\n\n        # Disable gradient computation\n        with torch.no_grad():\n\n            # Iterate over batches in the training set\n            for i, batch in enumerate(self._val_test_dl):\n                imgs, labels = batch\n                if self._cuda:\n                    imgs = imgs.cuda()\n                    labels = labels.cuda()\n                loss, preds = self.val_test_step(imgs, labels)\n                targets.extend(labels.cpu().detach().numpy())\n                predictions.extend(preds.cpu().detach().numpy())\n                total_loss += loss.item()\n\n        # Calculate average batch loss\n        avg_loss = total_loss / len(self._val_test_dl)\n\n        # Calculate custom metric\n        metric = self.metric_calc(np.array(targets), np.array(predictions))\n        print(f\"\\t Validation Loss: {avg_loss}, Validation Accuracy: {metric}\")\n\n        return avg_loss\n\n    def metric_calc(self, targets, predictions):\n        # Convert probabilities to predicted labels\n        predicted_labels = np.argmax(predictions, axis=1)\n\n        # Compare predicted labels with true labels\n        correct_predictions = np.sum(predicted_labels == np.argmax(targets, axis=1))\n\n        # Calculate accuracy\n        accuracy = correct_predictions / len(targets)\n\n        return accuracy\n\n    def fit(self, epochs=-1):\n        assert self._early_stopping_patience > 0 or epochs > 0\n        train_losses = []\n        val_losses = []\n        epoch = 1\n        no_improvements = 0\n        epoch_best = None\n\n        # Continue training until stopping condition is met\n        while True:\n\n            # Stop by epoch number\n            if epoch > epochs:\n                self.restore_best_checkpoint()\n                self.save_onnx('checkpoints/checkpoint_{:03d}b.onnx'.format(epoch_best))\n                print(f\"The lowest validation loss was achieved in epoch{epoch_best}\")\n                break\n            print(f'Epoch: {epoch}')\n\n            # Train the model for one epoch and record training and validation losses\n            train_losses.append(float(self.train_epoch()))\n            val_losses.append(float(self.val_test()))\n\n            # Adjust learning rate using scheduler if available\n            if self.scheduler:\n                self.scheduler.step(val_losses[-1])\n\n            # Save the checkpoint if validation loss is lowest\n            if val_losses[-1] == min(val_losses):\n                self.save_checkpoint()\n                no_improvements = 0\n                epoch_best = epoch\n            else:\n                no_improvements += 1\n\n            # Check if early stopping condition met\n            if no_improvements >= self._early_stopping_patience:\n                self.restore_best_checkpoint()\n                print('Early stopping.')\n                self.save_onnx('checkpoints/checkpoint_{:03d}b.onnx'.format(epoch_best))\n                print(f\"The lowest validation loss was achieved in epoch {epoch_best}\")\n                break\n            epoch += 1\n\n        return train_losses, val_losses\n\n    def lr_scheduler(self):\n        self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(self._optim, patience=5, verbose=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:11.988912Z","iopub.execute_input":"2024-05-06T16:10:11.989457Z","iopub.status.idle":"2024-05-06T16:10:12.019513Z","shell.execute_reply.started":"2024-05-06T16:10:11.989427Z","shell.execute_reply":"2024-05-06T16:10:12.018738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(f\"Code is running on {device}.\")\n\n# Set up data loading for the training, validation, and test sets\ntrain_data = CustomDataset(train_one_hot, 'training', root_dir + 'train/')\nval_data = CustomDataset(val_one_hot, 'validation', root_dir + 'train/')\n\ntrain_loader = DataLoader(train_data, batch_size=32, shuffle=True)\nval_loader = DataLoader(val_data, batch_size=32, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:12.020483Z","iopub.execute_input":"2024-05-06T16:10:12.020731Z","iopub.status.idle":"2024-05-06T16:10:12.038559Z","shell.execute_reply.started":"2024-05-06T16:10:12.020711Z","shell.execute_reply":"2024-05-06T16:10:12.037700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Transfer and Fine Tuning","metadata":{}},{"cell_type":"code","source":"model = models.inception_v3(pretrained=True)\nprint(model)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:12.039462Z","iopub.execute_input":"2024-05-06T16:10:12.039708Z","iopub.status.idle":"2024-05-06T16:10:12.436104Z","shell.execute_reply.started":"2024-05-06T16:10:12.039687Z","shell.execute_reply":"2024-05-06T16:10:12.435201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Freeze parameters of pretrained models\nfor param in model.parameters():    \n    param.requires_grad = False\n    \nmodel.fc = torch.nn.Sequential(\n    torch.nn.Linear(2048,120),\n    torch.nn.Softmax(dim=1)\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:12.439143Z","iopub.execute_input":"2024-05-06T16:10:12.439458Z","iopub.status.idle":"2024-05-06T16:10:12.447344Z","shell.execute_reply.started":"2024-05-06T16:10:12.439433Z","shell.execute_reply":"2024-05-06T16:10:12.446533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = torch.nn.CrossEntropyLoss()\noptimizer = optim.Adam(model.parameters(), lr=0.0006)\n\ntrainer = Trainer(model=model,\n                  crit=criterion,\n                  optim=optimizer,\n                  train_dl=train_loader,\n                  val_test_dl=val_loader,\n                  cuda=True,\n                  early_stopping_patience=30)\n\n# Call fit on trainer\nepochs = 50\ntrain_losses, val_losses = trainer.fit(epochs)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:10:12.448451Z","iopub.execute_input":"2024-05-06T16:10:12.448761Z","iopub.status.idle":"2024-05-06T16:33:13.792092Z","shell.execute_reply.started":"2024-05-06T16:10:12.448732Z","shell.execute_reply":"2024-05-06T16:33:13.791137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the training and validation losses\nplt.plot(np.arange(len(train_losses)), train_losses, label='train loss')\nplt.plot(np.arange(len(val_losses)), val_losses, label='val loss')\nplt.legend()\nplt.savefig('losses.png')","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:33:13.793721Z","iopub.execute_input":"2024-05-06T16:33:13.794108Z","iopub.status.idle":"2024-05-06T16:33:14.163579Z","shell.execute_reply.started":"2024-05-06T16:33:13.794081Z","shell.execute_reply":"2024-05-06T16:33:14.162705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test_loc = '/kaggle/input/dog-breed-identification/test/'\ntest_files = os.listdir(test_loc)\ntest_df = pd.DataFrame({'id': pd.Series(test_files).str.replace(\".jpg\",\"\")})\nprint(test_df.shape)\ntest_df.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:34:58.353136Z","iopub.execute_input":"2024-05-06T16:34:58.353530Z","iopub.status.idle":"2024-05-06T16:34:58.381381Z","shell.execute_reply.started":"2024-05-06T16:34:58.353500Z","shell.execute_reply":"2024-05-06T16:34:58.380414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test data loader\ntest_data = CustomDataset(test_df, 'test', root_dir + 'test/')\ntest_loader = DataLoader(test_data, batch_size=32, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:34:59.755293Z","iopub.execute_input":"2024-05-06T16:34:59.755645Z","iopub.status.idle":"2024-05-06T16:34:59.760727Z","shell.execute_reply.started":"2024-05-06T16:34:59.755620Z","shell.execute_reply":"2024-05-06T16:34:59.759793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():\n    model.cuda()\ntest_predictions = []\nwith torch.no_grad():\n    for inputs in test_loader:\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        test_predictions.extend(outputs.cpu().detach().numpy())\n        \npreds = pd.DataFrame(np.array(test_predictions), columns=list(train_one_hot.columns)[1:])","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:36:46.227360Z","iopub.execute_input":"2024-05-06T16:36:46.228452Z","iopub.status.idle":"2024-05-06T16:40:34.495410Z","shell.execute_reply.started":"2024-05-06T16:36:46.228417Z","shell.execute_reply":"2024-05-06T16:40:34.494392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:41:18.396442Z","iopub.execute_input":"2024-05-06T16:41:18.396795Z","iopub.status.idle":"2024-05-06T16:41:18.426478Z","shell.execute_reply.started":"2024-05-06T16:41:18.396769Z","shell.execute_reply":"2024-05-06T16:41:18.425540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.reset_index(drop=True)\nids = test_df.loc[:,'id']\ntest_predictions = pd.concat([ids, preds], axis=1)\nprint(test_predictions.shape)\ntest_predictions.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:43:04.452965Z","iopub.execute_input":"2024-05-06T16:43:04.453340Z","iopub.status.idle":"2024-05-06T16:43:04.488081Z","shell.execute_reply.started":"2024-05-06T16:43:04.453301Z","shell.execute_reply":"2024-05-06T16:43:04.487167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions.to_csv('submission.csv', index=None)","metadata":{"execution":{"iopub.status.busy":"2024-05-06T16:43:38.901242Z","iopub.execute_input":"2024-05-06T16:43:38.901615Z","iopub.status.idle":"2024-05-06T16:43:40.690493Z","shell.execute_reply.started":"2024-05-06T16:43:38.901587Z","shell.execute_reply":"2024-05-06T16:43:40.689670Z"},"trusted":true},"execution_count":null,"outputs":[]}]}