{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis notebook demonstrates how to train and tune a [convolutional neural network](https://en.wikipedia.org/wiki/Convolutional_neural_network) using [PyTorch](https://pytorch.org/) to classify images as dogs or cats.","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{"execution":{"iopub.status.busy":"2022-02-27T07:11:09.682162Z","iopub.execute_input":"2022-02-27T07:11:09.682451Z","iopub.status.idle":"2022-02-27T07:11:09.710079Z","shell.execute_reply.started":"2022-02-27T07:11:09.682379Z","shell.execute_reply":"2022-02-27T07:11:09.709103Z"}}},{"cell_type":"code","source":"from __future__ import annotations\n\nimport csv\nimport functools\nimport glob\nimport itertools\nimport os\nfrom collections import defaultdict\nfrom typing import Any\n\nimport albumentations as A\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport optuna\nimport pandas as pd\nfrom PIL import Image\nimport torch\nfrom torch import nn\nfrom torch import optim\nfrom torch.optim import lr_scheduler\nfrom torch.optim.lr_scheduler import _LRScheduler\nfrom torch.optim import Optimizer\nfrom torch.utils import data\nfrom tqdm import tqdm\nfrom rich import print\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Load rich extension for pretty output\n%load_ext rich","metadata":{"execution":{"iopub.status.busy":"2022-02-28T08:57:03.020548Z","iopub.execute_input":"2022-02-28T08:57:03.021145Z","iopub.status.idle":"2022-02-28T08:57:06.83269Z","shell.execute_reply.started":"2022-02-28T08:57:03.021068Z","shell.execute_reply":"2022-02-28T08:57:06.831818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuration\n\nThis is some basic configuration parameters that will be used throughout the notebook.","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"./data\"\nIMG_HEIGHT = IMG_WIDTH = 200\nEPOCHS = 20\nNUM_WORKERS = 4\nPIN_MEMORY = True\nLABEL_MAP = {\"cat\": 0, \"dog\": 1}\nFOLDS = 5\nCOSINE_ANNEALING_T0 = 10\nEARLY_STOPPING_ROUNDS = 5\nMODEL_DIR = \"./models\"\nINFERENCE_DIR = \"./inferences\"\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T08:57:06.834717Z","iopub.execute_input":"2022-02-28T08:57:06.834986Z","iopub.status.idle":"2022-02-28T08:57:06.885825Z","shell.execute_reply.started":"2022-02-28T08:57:06.834949Z","shell.execute_reply":"2022-02-28T08:57:06.884834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make model directory to save models\n# And inferences\nif not os.path.exists(MODEL_DIR):\n    os.mkdir(MODEL_DIR)\n    \nif not os.path.exists(INFERENCE_DIR):\n    os.mkdir(INFERENCE_DIR)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T08:57:06.887187Z","iopub.execute_input":"2022-02-28T08:57:06.887481Z","iopub.status.idle":"2022-02-28T08:57:06.895947Z","shell.execute_reply.started":"2022-02-28T08:57:06.88742Z","shell.execute_reply":"2022-02-28T08:57:06.895171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unzip Data and Resize Images","metadata":{}},{"cell_type":"code","source":"import os\nfrom zipfile import ZipFile\n\nif not os.path.exists(DATA_DIR):\n    os.mkdir(DATA_DIR)\n    \n    with ZipFile(\"../input/dogs-vs-cats/test1.zip\") as zipf:\n        zipf.extractall(DATA_DIR)\n\n    with ZipFile(\"../input/dogs-vs-cats/train.zip\") as zipf:\n        zipf.extractall(DATA_DIR)\n    \n    train_path = os.path.join(DATA_DIR, \"train\", \"*.jpg\")\n    test_path = os.path.join(DATA_DIR, \"test1\", \"*.jpg\")\n    for img_path in itertools.chain(glob.glob(train_path), glob.glob(test_path)):\n        img = Image.open(img_path)\n        if img.height != IMG_HEIGHT or img.width != IMG_WIDTH:\n            img = img.resize((IMG_WIDTH, IMG_HEIGHT), resample=Image.BILINEAR)\n            img.save(img_path)\n","metadata":{"execution":{"iopub.status.busy":"2022-02-28T08:57:06.898862Z","iopub.execute_input":"2022-02-28T08:57:06.899201Z","iopub.status.idle":"2022-02-28T09:00:23.367623Z","shell.execute_reply.started":"2022-02-28T08:57:06.899133Z","shell.execute_reply":"2022-02-28T09:00:23.366879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset And Data Functions\n\nThis section defines the `Dataset` that will be used by the model and some additional data functions.","metadata":{}},{"cell_type":"markdown","source":"The `CatsDogDataset` class implements a PyTorch Dataset to handle data loading.","metadata":{}},{"cell_type":"code","source":"class CatsDogsDataset(data.Dataset):\n    def __init__(\n        self,\n        csv: str,\n        transform: A.Compose = None,\n        labels: bool = True,\n    ):\n        self.df: pd.DataFrame = pd.read_csv(csv)\n        self.transform = transform\n        self.labels = labels\n\n    def __len__(self) -> int:\n        return len(self.df)\n\n    def __getitem__(self, idx: int) -> dict[str, torch.Tensor]:\n        # Load image path and label\n        if isinstance(idx, slice):\n            raise TypeError(f\"{self.__class__.__name__} doesn't support slicing.\")\n\n        img = self.df.iloc[idx, 0]\n\n        # Read image and convert to NumPy array\n        img = Image.open(img)\n        img = np.array(img)\n\n        # Apply transformations\n        if self.transform is not None:\n            img = self.transform(image=img)[\"image\"]\n\n        # Move channels in front\n        img = np.swapaxes(img, -1, 0)\n        img = torch.tensor(img, dtype=torch.float32)\n\n        sample = {\"image\": img}\n\n        # Add label to sample if required\n        if self.labels is True:\n            label = self.df.iloc[idx, 1]\n            sample[\"label\"] = torch.tensor(label, dtype=torch.float32)\n\n        return sample","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.369022Z","iopub.execute_input":"2022-02-28T09:00:23.369282Z","iopub.status.idle":"2022-02-28T09:00:23.378835Z","shell.execute_reply.started":"2022-02-28T09:00:23.369249Z","shell.execute_reply":"2022-02-28T09:00:23.377698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `dir_to_csv()` function takes either the train or test directories and makes a CSV file out of them.","metadata":{}},{"cell_type":"code","source":"def dir_to_csv(dir_name: str, dest: str, has_labels: bool = True) -> None:\n    def with_labels(path):\n        label_map = LABEL_MAP\n        yield from (\n            {\n                \"filename\": filename,\n                \"label\": label_map[\"cat\" if \"cat\" in filename else \"dog\"],\n            }\n            for filename in glob.glob(path)\n        )\n\n    path = os.path.join(DATA_DIR, dir_name, \"*.jpg\")\n    target = os.path.join(DATA_DIR, dest)\n\n    with open(target, mode=\"w+\") as f:\n\n        fieldnames = [\"filename\"]\n\n        if has_labels is True:\n            fieldnames.append(\"label\")\n            rows = with_labels(path)\n        else:\n            rows = ({\"filename\": filename} for filename in glob.glob(path))\n\n        writer = csv.DictWriter(f, fieldnames=fieldnames)\n        writer.writeheader()\n        writer.writerows(rows)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.380288Z","iopub.execute_input":"2022-02-28T09:00:23.380748Z","iopub.status.idle":"2022-02-28T09:00:23.392134Z","shell.execute_reply.started":"2022-02-28T09:00:23.380699Z","shell.execute_reply":"2022-02-28T09:00:23.391482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `order_test_data()` function orders the test data by the ","metadata":{}},{"cell_type":"code","source":"def order_test_data(csv_path: str) -> None:\n    path = os.path.join(DATA_DIR, csv_path)\n\n    df: pd.DataFrame = pd.read_csv(path)\n\n    target_names = []\n\n    for filename in df[\"filename\"]:\n        # Take out filename from full path\n        basename = os.path.basename(filename)\n\n        # Remove extension\n        name, _ = os.path.splitext(basename)\n\n        target_names.append(int(name))\n\n    df[\"target_name\"] = target_names\n\n    # Sort by IDs\n    df = df.sort_values(by=[\"target_name\"])\n    df = df.drop(\"target_name\", axis=1)\n\n    # Overwrite existing file\n    df.to_csv(path, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.394513Z","iopub.execute_input":"2022-02-28T09:00:23.395105Z","iopub.status.idle":"2022-02-28T09:00:23.402216Z","shell.execute_reply.started":"2022-02-28T09:00:23.395068Z","shell.execute_reply":"2022-02-28T09:00:23.40159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `get_transforms()` function returns an `albumentations.Compose` object with the transformations that should be applied on the images.","metadata":{}},{"cell_type":"code","source":"def get_transforms() -> A.Compose:\n    return A.Compose(\n        [\n            A.HorizontalFlip(p=0.4),\n            A.RandomBrightnessContrast(p=0.2),\n            A.Normalize(always_apply=True)\n        ]\n    )","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.403482Z","iopub.execute_input":"2022-02-28T09:00:23.403741Z","iopub.status.idle":"2022-02-28T09:00:23.414586Z","shell.execute_reply.started":"2022-02-28T09:00:23.4037Z","shell.execute_reply":"2022-02-28T09:00:23.413948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Class And Functions","metadata":{}},{"cell_type":"markdown","source":"The `CatsDogsModel` class implements a PyTorch model.\n\nIt takes the following inputs:\n\n* `in_channels`: Number of channels in the input images.\n* `conv_out_channels`: List of output channels for the convolutional layers. Thus, the model automatically adds as many convolutional layers as the length of the list with the specified number of output channels. Each convolutional layer is also followed by batch normalization, `ReLU` activaton and maxpooling of pool size `(2, 2)`.\n* `kernel_sizes`: List of kernel sizes for the convolutional layers. This should have the same length as the above list.\n* `linear_out_features`: Similar to `conv_out_channels` but for the linear layers, except the output layer. The model will have a total of one more linear layer than the length of this list. Each liniear layer is followed by a `ReLU` activation. \n* `n_targets`: Number of output features for the output linear layer.\n* `dropout`: Dropout probability for the optional dropout layer between the convolutional and linear parts. When `0.0`, no dropout layer is added to the model.\n* `op_activation`: Activation function for output layer. Can be `sigmoid` or `softmax`.\n\nDefining the model this way allows for convenient tuning of these values using some hyperparameter tuning library.","metadata":{}},{"cell_type":"code","source":"class CatsDogsModel(nn.Module):\n    op_activs = {\n        \"sigmoid\": nn.Sigmoid,\n        \"softmax\": nn.Softmax,\n    }\n    def __init__(\n        self,\n        conv_out_channels: list[int],\n        kernel_sizes: list[int],\n        linear_out_features: list[int], \n        n_targets: int = 1,\n        in_channels: int = 3,\n        dropout: float = 0.0,\n        op_activation: str = \"sigmoid\",\n    ) -> None:\n        super().__init__()\n\n        layers = []\n        in_c = in_channels\n\n        for out_channels, kernel_size in zip(conv_out_channels, kernel_sizes):\n            conv_block = self._make_conv_block(\n                in_channels=in_c,\n                out_channels=out_channels,\n                kernel_size=(kernel_size, kernel_size),\n            )\n            layers.append(conv_block)\n            in_c = out_channels\n\n        layers.append(nn.Flatten())\n\n        if dropout > 0.0:\n            layers.append(nn.Dropout(p=dropout))\n\n        in_features = self._get_linear_in_features(layers, in_channels)\n\n        for out_features in linear_out_features:\n            linear_block = self._make_linear_block(\n                in_features=in_features, out_features=out_features\n            )\n            layers.append(linear_block)\n            in_features = out_features\n\n        layers.append(nn.Linear(in_features, n_targets))\n        layers.append(self.op_activs[op_activation]())\n        self.layers = nn.Sequential(*layers)\n\n    def _make_conv_block(self, in_channels: int, out_channels: int, kernel_size: int) -> nn.Sequential:\n        return nn.Sequential(\n            nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, bias=False),\n            nn.BatchNorm2d(out_channels),\n            nn.ReLU(),\n            nn.MaxPool2d(kernel_size=(2, 2)),\n        )\n\n    def _get_linear_in_features(self, layers: list[nn.Module], in_channels: int = 3) -> int:\n        \"\"\"\n        Automatically calculate the in_features for the linear layer\n        Immediately after the final convolutional layer.\n        \"\"\"\n        x = torch.rand(1, in_channels, IMG_HEIGHT, IMG_WIDTH, dtype=torch.float32, device=DEVICE)\n        m = nn.Sequential(*layers)\n        m.to(DEVICE)\n\n        return m(x).size(-1)\n\n    def _make_linear_block(self, in_features: int, out_features: int) -> nn.Sequential:\n        return nn.Sequential(nn.Linear(in_features, out_features), nn.ReLU())\n\n    def forward(self, image: torch.Tensor) -> torch.Tensor:\n        return self.layers(image).squeeze(1)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.416024Z","iopub.execute_input":"2022-02-28T09:00:23.416461Z","iopub.status.idle":"2022-02-28T09:00:23.433715Z","shell.execute_reply.started":"2022-02-28T09:00:23.416425Z","shell.execute_reply":"2022-02-28T09:00:23.432832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"For example, to define a model with:\n\n* RGB input.\n* Four convolutional layers with output channels `16`, `32`, `32` and `64` and kernel sizes `3`, `3`, `5` and `5`.\n* Three linear layers with output features `64`, `32` and `16`.\n* A single Sigmoid output unit.\n* Dropout of `0.2`.\n\nYou can do the following:","metadata":{}},{"cell_type":"code","source":"m = CatsDogsModel(\n    conv_out_channels=[16, 32, 32, 64],\n    kernel_sizes=[3, 3, 5, 5],\n    linear_out_features=[64, 32, 16],\n    n_targets=1,\n    in_channels=3,\n    dropout=0.2,\n)\nm","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:23.436804Z","iopub.execute_input":"2022-02-28T09:00:23.437103Z","iopub.status.idle":"2022-02-28T09:00:31.304956Z","shell.execute_reply.started":"2022-02-28T09:00:23.437012Z","shell.execute_reply":"2022-02-28T09:00:31.304302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = torch.rand(1, 3, IMG_HEIGHT, IMG_WIDTH, dtype=torch.float32, device=DEVICE)\nm.to(DEVICE)\nm(x)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:31.306164Z","iopub.execute_input":"2022-02-28T09:00:31.306541Z","iopub.status.idle":"2022-02-28T09:00:32.132711Z","shell.execute_reply.started":"2022-02-28T09:00:31.306502Z","shell.execute_reply":"2022-02-28T09:00:32.132048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `get_model_params()` function takes a dictionary of parameters and extracts all parameters that are relevant to create a model, storing them in the appropriate format.","metadata":{}},{"cell_type":"code","source":"def get_model_params(params: dict[str, Any]) -> dict[str, Any]:\n    dict_params = {}\n\n    dict_params[\"conv_out_channels\"] = [\n        params[\"conv1_out\"],\n        params[\"conv2_out\"],\n        params[\"conv3_out\"],\n        params[\"conv4_out\"],\n    ]\n\n    k1, k2 = params[\"kernel_size1\"], params[\"kernel_size2\"]\n\n    dict_params[\"kernel_sizes\"] = [k1, k1, k2, k2]\n\n    dict_params[\"linear_out_features\"] = [\n        params[\"linear1_out\"],\n        params[\"linear2_out\"],\n        params[\"linear3_out\"],\n    ]\n\n    dict_params[\"dropout\"] = params[\"dropout\"]\n\n    return dict_params","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:32.134061Z","iopub.execute_input":"2022-02-28T09:00:32.134296Z","iopub.status.idle":"2022-02-28T09:00:32.141716Z","shell.execute_reply.started":"2022-02-28T09:00:32.134263Z","shell.execute_reply":"2022-02-28T09:00:32.139941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Engine Class And Training Functions","metadata":{}},{"cell_type":"markdown","source":"The `Engine` class handles training the model.","metadata":{}},{"cell_type":"code","source":"class Engine:\n    def __init__(\n        self, model: nn.Module, optimizer: Optimizer, scheduler: _LRScheduler = None\n    ) -> None:\n        self.model = model\n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        self.history = defaultdict(list)\n\n    @staticmethod\n    def loss_fn(y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:\n        fn = nn.BCELoss()\n        fn.to(DEVICE)\n        return fn(y_pred, y_true)\n\n    @staticmethod\n    def correct_and_accuracy(y_pred: torch.Tensor, y_true: torch.Tensor) -> tuple[int, float]:\n        correct = (y_true == (y_pred > 0.5).float()).float().sum()\n        return correct, correct / len(y_pred)\n            \n    def save_loss_curve(self, filename: str, best_iter: int = None) -> None:\n        train_loss = self.history[\"train_loss\"]\n        val_loss = self.history[\"val_loss\"]\n        \n        print(train_loss, val_loss)\n        \n        plt.figure()\n        plt.plot(train_loss, label=\"Training\")\n        plt.plot(val_loss, label=\"Validation\")\n        \n        if best_iter is not None:\n            plt.axvline(best_iter, color=\"black\", linestyle=\"--\")\n        \n        plt.xlabel(\"Epoch\")\n        plt.ylabel(\"Loss\")\n        plt.title(\"Loss Curve\")\n        plt.legend()\n        \n        path = os.path.join(MODEL_DIR, filename)\n        plt.savefig(path)\n        plt.clf()\n\n    def save_model(self, filename: str) -> None:\n        path = os.path.join(MODEL_DIR, filename)\n        torch.save(self.model.state_dict(), path)\n\n    def _train_one_step(\n        self, data: dict[str, torch.Tensor]\n    ) -> tuple[torch.Tensor, int, float]:\n        for key, value in data.items():\n            data[key] = value.to(DEVICE)\n\n        preds = self.model(image=data[\"image\"])\n\n        true = data[\"label\"]\n        loss = self.loss_fn(preds, true)\n\n        loss.backward()\n        self.optimizer.step()\n\n        correct, accuracy = self.correct_and_accuracy(preds, true)\n\n        return loss, correct, accuracy\n\n    def train(\n        self, data_loader: data.DataLoader, epoch_num: int = 0\n    ) -> tuple[float, float]:\n        self.model.train()\n        total_correct = 0\n\n        optimizer = self.optimizer\n        scheduler = self.scheduler\n\n        total_loss = accuracy = 0.0\n        iters = len(data_loader)\n        \n        with tqdm(data_loader, unit=\"batch\", desc=\"Training\") as p_loader:\n            for batch_index, data in enumerate(p_loader):\n                optimizer.zero_grad(set_to_none=True)\n\n                loss, correct, accuracy = self._train_one_step(data=data)\n                \n                total_correct += correct\n                loss = loss.item()\n                accuracy_per = accuracy * 100\n\n                postfix = {\"loss\": loss, \"batch acc\": f\"{accuracy_per: .2f}%\"}\n\n                postfix[\"lr\"] = (\n                    scheduler.get_last_lr()[0]\n                    if scheduler is not None\n                    else optimizer.param_groups[0][\"lr\"]\n                )\n\n                p_loader.set_postfix(postfix)\n\n                total_loss += loss\n\n                if scheduler is not None:\n                    scheduler.step(epoch_num + batch_index / iters)\n\n            avg_loss = total_loss / (batch_index + 1)\n            self.history[\"train_loss\"].append(avg_loss)\n            return avg_loss, total_correct\n\n    def _evaluate_one_step(\n        self, data: dict[str, torch.Tensor]\n    ) -> tuple[torch.Tensor, int, float]:\n        for key, value in data.items():\n            data[key] = value.to(DEVICE)\n\n        preds = self.model(image=data[\"image\"])\n\n        true = data[\"label\"]\n        loss = self.loss_fn(preds, true)\n        correct, accuracy = self.correct_and_accuracy(preds, true)\n\n        return loss, correct, accuracy\n\n    def evaluate(self, data_loader: data.DataLoader) -> tuple[float, float]:\n        self.model.eval()\n        \n        total_correct = 0\n        total_loss = accuracy = 0.0\n\n        with tqdm(data_loader, unit=\"batch\", desc=\"Validation\") as p_loader:\n            for batch_index, data in enumerate(p_loader):\n                with torch.no_grad():\n                    loss, correct, accuracy = self._evaluate_one_step(data=data)\n                    \n                total_correct += correct \n                loss = loss.item()\n                accuracy_per = accuracy * 100\n\n                postfix = {\"loss\": loss, \"batch acc\": f\"{accuracy_per: .2f}%\"}\n\n                p_loader.set_postfix(postfix)\n\n                total_loss += loss\n\n            avg_loss = total_loss / (batch_index + 1)\n            self.history[\"val_loss\"].append(avg_loss)\n            return avg_loss, total_correct","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:32.143256Z","iopub.execute_input":"2022-02-28T09:00:32.143727Z","iopub.status.idle":"2022-02-28T09:00:32.169966Z","shell.execute_reply.started":"2022-02-28T09:00:32.143688Z","shell.execute_reply":"2022-02-28T09:00:32.169231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The `train()` function runs the training loop. Optionally, it also saves the model and the loss curve of the model.","metadata":{}},{"cell_type":"code","source":"def init_weights(layer: nn.Module) -> None:\n    if isinstance(layer, (nn.Conv2d, nn.Linear)):\n        nn.init.xavier_normal_(layer.weight)\n\ndef train(\n    dataset: data.Dataset,\n    params: dict[str, Any],\n    *,\n    val_split: float = 0.2,\n    save_model: bool = False,\n) -> np.float32:\n    length = len(dataset)\n    val_len = int(val_split * length)\n    t_len = length - val_len\n    \n    generator = torch.Generator().manual_seed(42)\n    split = data.random_split(dataset, [t_len, val_len], generator=generator)\n\n    batch_size = params[\"batch_size\"]\n\n    train_loader = data.DataLoader(\n        split[0],\n        shuffle=True,\n        batch_size=batch_size,\n        num_workers=NUM_WORKERS,\n        pin_memory=PIN_MEMORY,\n    )\n\n    val_loader = data.DataLoader(\n        split[1],\n        shuffle=True,\n        batch_size=batch_size,\n        num_workers=NUM_WORKERS,\n        pin_memory=PIN_MEMORY,\n    )\n\n    model = CatsDogsModel(n_targets=1, **get_model_params(params))\n    model.apply(init_weights)\n    model.to(DEVICE)\n\n    optimizer = optim.Adam(model.parameters(), lr=params[\"lr\"])\n    scheduler = lr_scheduler.CosineAnnealingWarmRestarts(\n        optimizer=optimizer, T_0=COSINE_ANNEALING_T0\n    )\n\n    engine = Engine(model=model, optimizer=optimizer, scheduler=scheduler)\n\n    esr = params.get(\"early_stopping_rounds\", 5)\n    best_loss = np.inf\n    best_iter = counter = 0\n    losses = []\n\n    for epoch in range(EPOCHS):\n        print(f\"Epoch {epoch + 1}/{EPOCHS}\")\n\n        t_loss, t_correct = engine.train(data_loader=train_loader, epoch_num=epoch)\n        val_loss, val_correct = engine.evaluate(data_loader=val_loader)\n        \n        losses.append(val_loss)\n        \n        t_acc = t_correct / t_len\n        val_acc = val_correct / val_len\n        \n        print(f\"Avg. train loss={t_loss:.4f}, Train acc={t_acc:.4f}\")\n        print(f\"Avg. val loss={val_loss:.4f}, Val acc={val_acc:.4f}\")\n        \n        # Simple early stopping\n        if val_loss < best_loss:\n            best_loss, best_iter = val_loss, epoch\n            if save_model is True:\n                engine.save_model(f\"model-{epoch + 1}.pth\")\n        else:\n            counter += 1\n\n        if counter > esr:\n            break\n            \n    if save_model is True:\n        engine.save_loss_curve(f\"loss_curve.png\", best_iter)\n\n    return np.mean(losses, dtype=np.float32)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:01:04.874965Z","iopub.execute_input":"2022-02-28T09:01:04.875244Z","iopub.status.idle":"2022-02-28T09:01:04.891337Z","shell.execute_reply.started":"2022-02-28T09:01:04.875216Z","shell.execute_reply":"2022-02-28T09:01:04.890505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optuna Objective","metadata":{}},{"cell_type":"markdown","source":"Optuna requires an objective function which returns the metric to be optimized.","metadata":{}},{"cell_type":"code","source":"def objective(trial: optuna.trial.Trial, dataset: data.Dataset) -> np.float32:\n    params = {\n        \"conv1_out\": trial.suggest_int(\"conv1_out\", 16, 64, step=8),\n        \"conv2_out\": trial.suggest_int(\"conv2_out\", 16, 64, step=8),\n        \"conv3_out\": trial.suggest_int(\"conv3_out\", 16, 64, step=8),\n        \"conv4_out\": trial.suggest_int(\"conv4_out\", 16, 64, step=8),\n        \"linear1_out\": trial.suggest_int(\"linear1_out\", 16, 64, step=8),\n        \"linear2_out\": trial.suggest_int(\"linear2_out\", 16, 64, step=8),\n        \"linear3_out\": trial.suggest_int(\"linear3_out\", 16, 64, step=8),\n        \"kernel_size1\": trial.suggest_categorical(\"kernel_size1\", [3, 5, 7]),\n        \"kernel_size2\": trial.suggest_categorical(\"kernel_size2\", [3, 5, 7]),\n        \"lr\": trial.suggest_float(\"lr\", 1e-4, 1e-1, log=True),\n        \"batch_size\": trial.suggest_categorical(\"batch_size\", [32, 64, 128]),\n        \"dropout\": trial.suggest_uniform(\"dropout\", 0.1, 0.7),\n        \"early_stopping_rounds\": trial.suggest_categorical(\"early_stopping_rounds\", [5, 10, 15, 20])\n    }\n\n    return train(dataset, params)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:50.606308Z","iopub.execute_input":"2022-02-28T09:00:50.606561Z","iopub.status.idle":"2022-02-28T09:00:50.616465Z","shell.execute_reply.started":"2022-02-28T09:00:50.606532Z","shell.execute_reply":"2022-02-28T09:00:50.615699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"markdown","source":"# Convert Directories To CSV","metadata":{}},{"cell_type":"code","source":"dir_to_csv(\"train\", \"train_data.csv\")\ndir_to_csv(\"test1\", \"test_data.csv\", has_labels=False)\norder_test_data(\"test_data.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:32.201071Z","iopub.execute_input":"2022-02-28T09:00:32.201321Z","iopub.status.idle":"2022-02-28T09:00:32.540369Z","shell.execute_reply.started":"2022-02-28T09:00:32.201286Z","shell.execute_reply":"2022-02-28T09:00:32.539682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed","metadata":{}},{"cell_type":"code","source":"torch.manual_seed(42)\ntorch.backends.cudnn.benchmark = True","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:32.541553Z","iopub.execute_input":"2022-02-28T09:00:32.541816Z","iopub.status.idle":"2022-02-28T09:00:32.547336Z","shell.execute_reply.started":"2022-02-28T09:00:32.541783Z","shell.execute_reply":"2022-02-28T09:00:32.546646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read and Create Dataset","metadata":{}},{"cell_type":"code","source":"dataset_path = os.path.join(DATA_DIR, \"train_data.csv\")\ntransform = get_transforms()\ndataset = CatsDogsDataset(csv=dataset_path, transform=transform)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:00:32.5484Z","iopub.execute_input":"2022-02-28T09:00:32.548674Z","iopub.status.idle":"2022-02-28T09:00:32.572554Z","shell.execute_reply.started":"2022-02-28T09:00:32.548616Z","shell.execute_reply":"2022-02-28T09:00:32.571939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tune Model Using Optuna","metadata":{}},{"cell_type":"code","source":"objective_ = functools.partial(objective, dataset=dataset)\n\npruner = optuna.pruners.SuccessiveHalvingPruner()\nsampler = optuna.samplers.TPESampler(seed=42, multivariate=True)\nstudy = optuna.create_study(\n    direction=\"minimize\",\n    pruner=pruner,\n    sampler=sampler\n)\nstudy.optimize(objective_, n_trials=20, gc_after_trial=True)","metadata":{"execution":{"iopub.status.busy":"2022-02-28T09:01:07.988171Z","iopub.execute_input":"2022-02-28T09:01:07.98874Z","iopub.status.idle":"2022-02-28T09:02:17.844626Z","shell.execute_reply.started":"2022-02-28T09:01:07.988701Z","shell.execute_reply":"2022-02-28T09:02:17.843512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Final Model Using Best Parameters","metadata":{}},{"cell_type":"code","source":"print(study.best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T17:30:52.566838Z","iopub.execute_input":"2022-02-27T17:30:52.567289Z","iopub.status.idle":"2022-02-27T17:30:52.578021Z","shell.execute_reply.started":"2022-02-27T17:30:52.567252Z","shell.execute_reply":"2022-02-27T17:30:52.577127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_trial = study.best_trial\nval_loss = train(dataset=dataset, params=best_trial.params, save_model=True)\nprint(f\"Avg. validation loss:{val_loss:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-27T17:30:55.511415Z","iopub.execute_input":"2022-02-27T17:30:55.512124Z","iopub.status.idle":"2022-02-27T17:31:53.694906Z","shell.execute_reply.started":"2022-02-27T17:30:55.512072Z","shell.execute_reply":"2022-02-27T17:31:53.69416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Loss curve","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 10))\n\npath = os.path.join(MODEL_DIR, \"loss_curve.png\")\nim = plt.imread(path)\nax.imshow(im)\nax.set_axis_off()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-27T17:40:35.949641Z","iopub.execute_input":"2022-02-27T17:40:35.950471Z","iopub.status.idle":"2022-02-27T17:40:36.125368Z","shell.execute_reply.started":"2022-02-27T17:40:35.95043Z","shell.execute_reply":"2022-02-27T17:40:36.124496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"def make_inference(csv_file: str, params: dict[str, Any]) -> None:\n    transform = A.Compose([A.Normalize(always_apply=True)])\n\n    test_data_path = os.path.join(DATA_DIR, csv_file)\n\n    test_data = CatsDogsDataset(\n        csv=test_data_path,\n        transform=transform,\n        labels=False,\n    )\n     \n    models_path = os.path.join(MODEL_DIR, \"*.pth\")\n    \n    for model_path in glob.glob(models_path):\n        model = CatsDogsModel(**get_model_params(params), n_targets=1)\n\n        state_dict = torch.load(model_path)\n        model.load_state_dict(state_dict)\n        model.to(DEVICE)\n        model.eval()\n\n        data_loader = data.DataLoader(\n            test_data,\n            batch_size=params[\"batch_size\"],\n            num_workers=NUM_WORKERS,\n            pin_memory=PIN_MEMORY\n        )\n\n        df_dict = defaultdict(list)\n        df_dict[\"id\"] = range(1, len(test_data) + 1)\n\n        for idx, batch in enumerate(data_loader, 1):\n            for k, v in batch.items():\n                batch[k] = v.to(DEVICE)\n\n            with torch.no_grad():\n                preds = model(image=batch[\"image\"])\n            \n            preds = (preds > 0.5).int()\n            df_dict[\"label\"].extend(preds.tolist())\n            \n        basename = os.path.basename(model_path)\n        filename, _ = os.path.splitext(basename)\n\n        df = pd.DataFrame.from_dict(df_dict)\n        path = os.path.join(INFERENCE_DIR, f\"{filename}_submission.csv\")\n        df.to_csv(path, index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:00:27.361911Z","iopub.execute_input":"2022-02-27T18:00:27.36258Z","iopub.status.idle":"2022-02-27T18:00:27.373635Z","shell.execute_reply.started":"2022-02-27T18:00:27.362539Z","shell.execute_reply":"2022-02-27T18:00:27.372842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_inference(\"test_data.csv\", best_trial.params)","metadata":{"execution":{"iopub.status.busy":"2022-02-27T18:00:29.504013Z","iopub.execute_input":"2022-02-27T18:00:29.504267Z","iopub.status.idle":"2022-02-27T18:00:54.162003Z","shell.execute_reply.started":"2022-02-27T18:00:29.504238Z","shell.execute_reply":"2022-02-27T18:00:54.161065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}