{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Import required dependencies.","metadata":{}},{"cell_type":"code","source":"import os\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport zipfile\n\nfrom PIL import Image\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom typing import Union","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:07:41.117484Z","iopub.execute_input":"2022-10-31T23:07:41.118156Z","iopub.status.idle":"2022-10-31T23:07:43.961193Z","shell.execute_reply.started":"2022-10-31T23:07:41.118043Z","shell.execute_reply":"2022-10-31T23:07:43.960114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get data and define constants.","metadata":{}},{"cell_type":"code","source":"with zipfile.ZipFile('/kaggle/input/dogs-vs-cats/train.zip',\"r\") as train:\n    train.extractall()\n\ntrain_dir = './train'\ntrain_files = os.listdir(train_dir)\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:08:29.683180Z","iopub.execute_input":"2022-10-31T23:08:29.683736Z","iopub.status.idle":"2022-10-31T23:08:42.868684Z","shell.execute_reply.started":"2022-10-31T23:08:29.683702Z","shell.execute_reply":"2022-10-31T23:08:42.867699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define custom Dataset.","metadata":{}},{"cell_type":"code","source":"class CatDogDataset(Dataset):\n    \"\"\"Create Pytorch Dataset for cat/dog binary classification.\"\"\"\n    def __init__(self, file_list, file_dir, transform):\n        \"\"\"\n        Init method.\n        \n        Parameters\n        ----------\n        file_list : List[str]\n            List with names of the files. Must include dog or cat in the name.\n        file_dir : str\n            Directory path.\n        transform : Callable\n            Transformations to apply to tensors.\n        \"\"\"\n        self.file_list = file_list\n        self.file_dir = file_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.file_list)\n    \n    def __getitem__(self, idx):\n        img = Image.open(os.path.join(self.file_dir, self.file_list[idx]))\n        if 'dog' in self.file_list[idx]:\n            label = 1.0\n        else:\n            label = 0.0\n        \n        img = self.transform(img)\n        return {\"image\": img, \"target\": torch.tensor([label])}","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:28:12.618647Z","iopub.execute_input":"2022-10-31T23:28:12.619013Z","iopub.status.idle":"2022-10-31T23:28:12.627032Z","shell.execute_reply.started":"2022-10-31T23:28:12.618983Z","shell.execute_reply":"2022-10-31T23:28:12.625872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define AlexNet from scratch. \nFor more info about it, please refer to https://en.wikipedia.org/wiki/AlexNet","metadata":{}},{"cell_type":"code","source":"class AlexNet(nn.Module):\n    \"\"\"\n    Neural net that mimics AlexNet from scratch.\n\n    Attributes\n    ----------\n    ConvolutedBlock\n        Long short term memory (LSTM) block that process user sequences with an input shape\n        (batch_size, num_sequences, logs_features) which returns a 3-dimensional tensor.\n    DenseBlock\n        Dense block of linear layers that process users profiles with an input shape of (batch_size, users_features)\n        and returns a 2-dimensional tensor.\n\n    Notes\n    -----\n    - The architecture follows the exact same structure than AlexNet (see: https://en.wikipedia.org/wiki/AlexNet).\n    - Some output channels and output features have been reduced in size due to performance improvements.\n    - We don't apply a sigmoid activation in the last layer. This is because we are using\n    torch.nn.functional.binary_cross_entropy_with_logits as the current loss which proves to be more numerically stable.\n\n    Parameters\n    ----------\n    nn : torch.nn.Module\n        Base class for all neural network modules. All pytorch models should be a subclass this class.\n    \"\"\"\n    \n    def __init__(self):\n        \"\"\"Init method.\"\"\"\n        super(AlexNet, self).__init__()\n        self.ConvolutedBlock = nn.Sequential(\n            nn.Conv2d(in_channels=3, out_channels=96, kernel_size=11, stride=4, padding=0),\n            nn.BatchNorm2d(96),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(3,3), stride=2),\n            nn.Conv2d(in_channels=96, out_channels=256, kernel_size=5, stride=1, padding=2),\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(3,3), stride=2),            \n            nn.Conv2d(in_channels=256, out_channels=384, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(384),\n            nn.ReLU(),            \n            nn.Conv2d(in_channels=384, out_channels=256, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(256),\n            nn.ReLU(),\n            nn.Conv2d(in_channels=256, out_channels=96, kernel_size=3, stride=1, padding=1),\n            nn.BatchNorm2d(96),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(3,3), stride=2)\n        )\n        \n        self.DenseBlock = nn.Sequential(\n            nn.Dropout(0.5),\n            nn.Linear(in_features=3456, out_features=512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(in_features=512, out_features=248),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(in_features=248, out_features=1)\n        )\n        \n    def forward(self, x: torch.tensor):\n        \"\"\"Forward pass.\"\"\"\n        x = self.ConvolutedBlock(x)\n        x = torch.flatten(x, 1)\n        x = self.DenseBlock(x)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:08:46.094778Z","iopub.execute_input":"2022-10-31T23:08:46.095203Z","iopub.status.idle":"2022-10-31T23:08:46.107431Z","shell.execute_reply.started":"2022-10-31T23:08:46.095171Z","shell.execute_reply":"2022-10-31T23:08:46.106325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define model trainer.","metadata":{}},{"cell_type":"code","source":"class Engine:\n    \"\"\"\n    Pytorch engine to train neural network.\n\n    Attributes\n    ----------\n    model : AlexNet\n        Custom model in use.\n    loss_fn : BCEWithLogitsLoss\n        Type of loss function in use. Currently binary cross entropy with logits.\n    optimizer : Union[RMSprop, Adam]\n        Optimizer to use. Either RMSprop or Adam.\n    device : str\n        Device on where to perform computations (either 'cuda' or 'cpu').\n    loss_history : Dict[str, List[float]]\n        Dictionary with (\"train\", \"val\") as keys and list with loss per epoch as values.\n    acc_history : Dict[str, List[float]]\n        Dictionary with (\"val\") as keys and list with accuracy per epoch as values.\n    \"\"\"\n\n    def __init__(self, model: AlexNet, loss_fn: nn.BCEWithLogitsLoss, optimizer: Union[optim.RMSprop, optim.Adam], device: str):\n        self.model = model.to(device)\n        self.loss_fn = loss_fn\n        self.optimizer = optimizer\n        self.device = device\n        self.loss_history = {\"train\": [], \"val\": []}\n        self.acc_history = {\"val\" : []}\n\n    def _train_step(self, batch: DataLoader):\n        \"\"\"Run backprop, update model and return loss.\"\"\"\n        self.model.train()\n        # Get tensors.\n        image = batch[\"image\"].to(self.device)\n        targets = batch[\"target\"].to(self.device)\n        # Clean gradients and perform backwards pass.\n        self.optimizer.zero_grad()\n        logits = self.model(image)\n        loss = self.loss_fn(logits, targets)\n        loss.backward()\n        # Update model.\n        self.optimizer.step()\n        \n        return loss.item()\n\n    def _validation_step(self, batch: DataLoader):\n        \"\"\"Compute loss for val data.\"\"\"\n        self.model.eval()\n        with torch.no_grad():\n            # Get tensors.\n            image = batch[\"image\"].to(self.device)\n            targets = batch[\"target\"].to(self.device)\n            # Get logits and compute loss.\n            logits = self.model(image)\n            loss = self.loss_fn(logits, targets).item()\n            total = targets.size(0)\n            correct = (logits.sigmoid().round() == targets).sum().item()\n\n            return loss, total, correct\n\n    def train(self, train_loader: DataLoader, val_loader: DataLoader, n_epochs: int):\n        \"\"\"\n        Train neural network and display training and validation loss.\n\n        Parameters\n        ----------\n        train_loader : DataLoader\n            DataLoader with training data.\n        val_loader : DataLoader\n            DataLoader with validation data.\n        n_epochs : int\n            Number of epochs to train.\n        \"\"\"\n        for epoch in range(1, n_epochs + 1):\n            losses = []\n\n            for batch in train_loader:\n                loss = self._train_step(batch)\n                losses.append(loss)\n\n            epoch_train_loss = np.mean(losses)\n            self.loss_history[\"train\"].append(epoch_train_loss)\n\n            with torch.no_grad():\n                losses, accuracies = [], []\n                correct = 0\n                total = 0\n\n                for batch in val_loader:\n                    loss, tot, corr = self._validation_step(batch)\n                    losses.append(loss)\n                    total += tot\n                    correct += corr\n\n                epoch_val_loss = np.mean(losses)\n                epoch_val_acc = np.mean(accuracies)\n                self.loss_history[\"val\"].append(epoch_val_loss)\n                \n                self.acc_history[\"val\"].append(epoch_val_acc)\n\n            print(\n                f\"[{epoch}/{n_epochs}] Training loss: {epoch_train_loss:.4f}\\t Validation loss: {epoch_val_loss:.4f}\\t Validation accuracy: {correct/total:.4f}\"\n            )\n\n    def plot_losses(self):\n        \"\"\"Plot losses accross epochs.\"\"\"\n        plt.plot(self.loss_history[\"train\"], label=\"Training loss\")\n        plt.plot(self.loss_history[\"val\"], label=\"Validation loss\")\n        plt.legend()\n        plt.title(\"Losses\")\n        plt.show()\n        plt.close()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:08:49.484597Z","iopub.execute_input":"2022-10-31T23:08:49.484976Z","iopub.status.idle":"2022-10-31T23:08:49.500688Z","shell.execute_reply.started":"2022-10-31T23:08:49.484943Z","shell.execute_reply":"2022-10-31T23:08:49.499580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_epochs = 20\nlearning_rate = 5e-4\nweight_decay = 5e-4\nbatch_size = 64\n\ndata_transform = transforms.Compose([\n    transforms.Resize(227),\n    transforms.ColorJitter(),\n    transforms.RandomCrop(227),\n    transforms.RandomHorizontalFlip(),\n    transforms.Resize(227),\n    transforms.ToTensor()\n])\n\ntrain_files, val_files = train_test_split(train_files)\n\ntrain_data = CatDogDataset(train_files, train_dir, data_transform)\nval_data = CatDogDataset(val_files, train_dir, data_transform)\n\ntrain_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(val_data, batch_size=batch_size, shuffle=True)\n\nmodel = AlexNet()\nloss_fn = nn.BCEWithLogitsLoss()\noptimizer = optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\nopt = Engine(model=model, loss_fn=loss_fn, optimizer=optimizer, device=device)\nopt.train(train_loader, val_loader, n_epochs)\nopt.plot_losses()","metadata":{"execution":{"iopub.status.busy":"2022-10-31T23:31:18.570570Z","iopub.execute_input":"2022-10-31T23:31:18.570942Z","iopub.status.idle":"2022-10-31T23:38:39.480591Z","shell.execute_reply.started":"2022-10-31T23:31:18.570912Z","shell.execute_reply":"2022-10-31T23:38:39.478890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}