{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7128545,"sourceType":"datasetVersion","datasetId":4048250},{"sourceId":144358840,"sourceType":"kernelVersion"},{"sourceId":149254949,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"See the following repositories for details  \nhttps://github.com/tubo213/kaggle-child-mind-institute-detect-sleep-states","metadata":{}},{"cell_type":"markdown","source":"# Install additional libraries","metadata":{}},{"cell_type":"code","source":"import time\nimport psutil\nimport os\n\nimport torch\nimport torch.nn as nn\nfrom abc import abstractmethod\n\nfrom typing import Optional","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:11.356706Z","iopub.execute_input":"2023-12-03T22:22:11.357675Z","iopub.status.idle":"2023-12-03T22:22:11.362340Z","shell.execute_reply.started":"2023-12-03T22:22:11.357637Z","shell.execute_reply":"2023-12-03T22:22:11.361203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install hydra-core segmentation_models_pytorch==0.3.3 --no-index --find-links=/kaggle/input/ex-library\nimport segmentation_models_pytorch as smp","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:11.368809Z","iopub.execute_input":"2023-12-03T22:22:11.369162Z","iopub.status.idle":"2023-12-03T22:22:23.174333Z","shell.execute_reply.started":"2023-12-03T22:22:11.369123Z","shell.execute_reply":"2023-12-03T22:22:23.173305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/cmi-code/kaggle-child-mind-institute-detect-sleep-states/","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:23.176204Z","iopub.execute_input":"2023-12-03T22:22:23.176577Z","iopub.status.idle":"2023-12-03T22:22:23.184207Z","shell.execute_reply.started":"2023-12-03T22:22:23.176544Z","shell.execute_reply":"2023-12-03T22:22:23.183059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Config\nDURATION = 5760\nDOWNSAMPLE_RATE = 4\nMODEL_NAME=\"resnet34\"\nPHASE = 'test'\nEXP_NAME = 'submit-55/submit-55'","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:23.185429Z","iopub.execute_input":"2023-12-03T22:22:23.185739Z","iopub.status.idle":"2023-12-03T22:22:23.194668Z","shell.execute_reply.started":"2023-12-03T22:22:23.185709Z","shell.execute_reply":"2023-12-03T22:22:23.193896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"code","source":"!python -m run.prepare_data dir=kaggle phase=$PHASE","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:23.197281Z","iopub.execute_input":"2023-12-03T22:22:23.197838Z","iopub.status.idle":"2023-12-03T22:22:25.515782Z","shell.execute_reply.started":"2023-12-03T22:22:23.197812Z","shell.execute_reply":"2023-12-03T22:22:25.514589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"from typing import Callable, Optional\n# ref: https://github.com/bamps53/kaggle-dfl-3rd-place-solution/blob/master/models/cnn_3d.py\nfrom functools import partial\nfrom typing import Optional\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom dataclasses import dataclass\n\n@dataclass\nclass ModelOutput:\n    logits: torch.Tensor\n    loss: Optional[torch.Tensor] = None\n    preds: Optional[torch.Tensor] = None\n    labels: Optional[torch.Tensor] = None\n        \n        \ndef get_rand_1dbbox(n_timesteps: int, lam: float) -> tuple[int, int]:\n    \"\"\"Get random 1D bounding box.\n\n    Args:\n        n_timesteps (int): Number of timesteps.\n        lam (float): Lambda value.\n\n    Returns:\n        tuple[int, int]: (start, end) of the bounding box.\n    \"\"\"\n    cut_rat = np.sqrt(1.0 - lam)\n    cut_len = int(n_timesteps * cut_rat)\n\n    start = np.random.randint(0, n_timesteps - cut_len)\n    end = start + cut_len\n\n    return start, end\n\nclass BaseModel(nn.Module):\n    def __init__(self):\n        super().__init__()\n\n    def forward(\n        self,\n        x: torch.Tensor,\n        labels: Optional[torch.Tensor] = None,\n        do_mixup: bool = False,\n        do_cutmix: bool = False,\n    ):\n        \"\"\"Forward pass of the model.\n\n        Args:\n            x (torch.Tensor): (batch_size, n_channels, n_timesteps)\n            labels (Optional[torch.Tensor], optional): (batch_size, n_timesteps, n_classes)\n            do_mixup (bool, optional): Defaults to False.\n            do_cutmix (bool, optional):  Defaults to False.\n\n        Returns:\n            ModelOutput: model output\n        \"\"\"\n        if labels is not None:\n            logits, labels = self._forward(x, labels, do_mixup, do_cutmix)\n            loss = self.loss_fn(logits, labels)\n            return ModelOutput(logits=logits, loss=loss, labels=labels)\n        else:\n            logits = self._forward(x, labels=None, do_mixup=False, do_cutmix=False)\n            if isinstance(logits, torch.Tensor):\n                return ModelOutput(logits=logits)\n            else:\n                raise ValueError(\"logits must be a torch.Tensor\")\n\n    @torch.no_grad()\n    def predict(\n        self,\n        x: torch.Tensor,\n        org_duration: int,\n        labels: Optional[torch.Tensor] = None,\n    ):\n        output = self.forward(x, labels, False, False)\n        preds = self._logits_to_proba_per_step(output.logits, org_duration)\n        output.preds = preds\n\n        if labels is not None:\n            labels = self._correct_labels(labels, org_duration)\n            output.labels = labels\n\n        return output\n\n    @abstractmethod\n    def _forward(\n        self,\n        x: torch.Tensor,\n        labels: Optional[torch.Tensor] = None,\n        do_mixup: bool = False,\n        do_cutmix: bool = False,\n    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Forward pass of the model.\n\n        Args:\n            x (torch.Tensor): (batch_size, n_channels, n_timesteps)\n            labels (Optional[torch.Tensor], optional): (batch_size, n_timesteps, n_classes)\n            do_mixup (bool, optional): Defaults to False.\n            do_cutmix (bool, optional):  Defaults to False.\n\n        Returns:\n            torch.Tensor | tuple[torch.Tensor, torch.Tensor]: logits or (logits, labels)\n        \"\"\"\n        raise NotImplementedError\n\n    @abstractmethod\n    def _logits_to_proba_per_step(\n        self,\n        logits: torch.Tensor,\n        org_duration: int,\n    ) -> torch.Tensor:\n        \"\"\"Convert logits to probabilities per step.\n\n        Args:\n            logits (torch.Tensor): (batch_size, n_timesteps, n_classes)\n            org_duration (int): original duration in seconds\n\n        Returns:\n            torch.Tensor: (batch_size, org_duration, n_classes)\n        \"\"\"\n        raise NotImplementedError\n\n    @abstractmethod\n    def _correct_labels(\n        self,\n        labels: torch.Tensor,\n        org_duration: int,\n    ) -> torch.Tensor:\n        \"\"\"Correct labels to match the output of the model.\n\n        Args:\n            labels (torch.Tensor): (batch_size, org_duration, n_classes)\n            org_duration (int): original duration in seconds\n\n        Returns:\n            torch.Tensor: (batch_size, org_duration, n_classes)\n        \"\"\"\n        raise NotImplementedError\nclass Cutmix:\n    def __init__(self, alpha: float = 0.4):\n        self.alpha = alpha\n\n    def __call__(\n        self, imgs: torch.Tensor, labels: torch.Tensor\n    ) -> tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Cutmix augmentation.\n\n        Args:\n            imgs (torch.Tensor): (batch_size, n_channels, n_timesteps)\n            labels (torch.Tensor): (batch_size, n_timesteps, n_classes)\n\n        Returns:\n            tuple[torch.Tensor]: mixed_imgs (batch_size, n_channels, n_timesteps)\n                                 mixed_labels (batch_size, n_timesteps, n_classes)\n        \"\"\"\n        batch_size = imgs.size(0)\n        idx = torch.randperm(batch_size)\n\n        shuffled_imgs = imgs[idx]\n        shuffled_labels = labels[idx]\n\n        lam = np.random.beta(self.alpha, self.alpha)\n        start, end = get_rand_1dbbox(imgs.size(2), lam)\n\n        mixed_imgs = torch.concatenate(\n            [imgs[:, :, :start], shuffled_imgs[:, :, start:end], imgs[:, :, end:]], dim=2\n        )\n        mixed_labels = torch.concatenate(\n            [labels[:, :start, :], shuffled_labels[:, start:end, :], labels[:, end:, :]], dim=1\n        )\n\n        return mixed_imgs, mixed_labels\n    \nclass Mixup:\n    def __init__(self, alpha: float = 0.4):\n        self.alpha = alpha\n\n    def __call__(\n        self, imgs: torch.Tensor, labels: torch.Tensor\n    ) -> tuple[torch.Tensor, torch.Tensor]:\n        \"\"\"Mixup augmentation.\n\n        Args:\n            imgs (torch.Tensor): (batch_size, n_channels, n_timesteps)\n            labels (torch.Tensor): (batch_size, n_timesteps, n_classes)\n\n        Returns:\n            tuple[torch.Tensor]: mixed_imgs (batch_size, n_channels, n_timesteps)\n                                 mixed_labels (batch_size, n_timesteps, n_classes)\n        \"\"\"\n        batch_size = imgs.size(0)\n        idx = torch.randperm(batch_size)\n        lam = np.random.beta(self.alpha, self.alpha)\n\n        mixed_imgs: torch.Tensor = lam * imgs + (1 - lam) * imgs[idx]\n        mixed_labels: torch.Tensor = lam * labels + (1 - lam) * labels[idx]\n\n        return mixed_imgs, mixed_labels\n\nclass Spec2DCNN(BaseModel):\n    def __init__(\n        self,\n        feature_extractor: nn.Module,\n        decoder: nn.Module,\n        encoder_name: str,\n        in_channels: int,\n        encoder_weights: Optional[str] = None,\n        mixup_alpha: float = 0.5,\n        cutmix_alpha: float = 0.5,\n    ):\n        super().__init__()\n        self.feature_extractor = feature_extractor\n        self.encoder = smp.Unet(\n            encoder_name=encoder_name,\n            encoder_weights=encoder_weights,\n            in_channels=in_channels,\n            classes=1,\n        )\n        self.decoder = decoder\n        self.mixup = Mixup(mixup_alpha)\n        self.cutmix = Cutmix(cutmix_alpha)\n        self.loss_fn = nn.BCEWithLogitsLoss()\n\n    def _forward(\n        self,\n        x: torch.Tensor,\n        labels: Optional[torch.Tensor] = None,\n        do_mixup: bool = False,\n        do_cutmix: bool = False,\n    ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:\n        x = self.feature_extractor(x)  # (batch_size, n_channels, height, n_timesteps)\n\n        if do_mixup and labels is not None:\n            x, labels = self.mixup(x, labels)\n        if do_cutmix and labels is not None:\n            x, labels = self.cutmix(x, labels)\n\n        x = self.encoder(x).squeeze(1)  # (batch_size, height, n_timesteps)\n        logits = self.decoder(x)  # (batch_size, n_timesteps, n_classes)\n\n        if labels is not None:\n            return logits, labels\n        else:\n            return logits\n\n    def _logits_to_proba_per_step(self, logits: torch.Tensor, org_duration: int) -> torch.Tensor:\n        preds = logits.sigmoid()\n        return resize(preds, size=[org_duration, preds.shape[-1]], antialias=False)[:, :, [1, 2]]\n\n    def _correct_labels(self, labels: torch.Tensor, org_duration: int) -> torch.Tensor:\n        return resize(labels, size=[org_duration, labels.shape[-1]], antialias=False)[:, :, [1, 2]]\n\n\nclass SEModule(nn.Module):\n    def __init__(self, channel, reduction=8):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel, bias=False),\n            nn.Sigmoid(),\n        )\n\n    def forward(self, x):\n        (\n            b,\n            c,\n            _,\n        ) = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1)\n        return x * y.expand_as(x)\n\n\nclass DoubleConv(nn.Module):\n    \"\"\"(convolution => [BN] => ReLU) * 2\"\"\"\n\n    def __init__(\n        self,\n        in_channels,\n        out_channels,\n        mid_channels=None,\n        norm=nn.BatchNorm1d,\n        se=False,\n        res=False,\n    ):\n        super().__init__()\n        self.res = res\n        if not mid_channels:\n            mid_channels = out_channels\n        if se:\n            non_linearity = SEModule(out_channels)\n        else:\n            non_linearity = nn.ReLU(inplace=True)\n        self.double_conv = nn.Sequential(\n            nn.Conv1d(in_channels, mid_channels, kernel_size=3, padding=1, bias=False),\n            norm(mid_channels),\n            nn.ReLU(inplace=True),\n            nn.Conv1d(mid_channels, out_channels, kernel_size=3, padding=1, bias=False),\n            norm(out_channels),\n            non_linearity,\n        )\n\n    def forward(self, x):\n        if self.res:\n            return x + self.double_conv(x)\n        else:\n            return self.double_conv(x)\n\n\nclass Down(nn.Module):\n    \"\"\"Downscaling with maxpool then double conv\"\"\"\n\n    def __init__(\n        self, in_channels, out_channels, scale_factor, norm=nn.BatchNorm1d, se=False, res=False\n    ):\n        super().__init__()\n        self.maxpool_conv = nn.Sequential(\n            nn.MaxPool1d(scale_factor),\n            DoubleConv(in_channels, out_channels, norm=norm, se=se, res=res),\n        )\n\n    def forward(self, x):\n        return self.maxpool_conv(x)\n\n\nclass Up(nn.Module):\n    \"\"\"Upscaling then double conv\"\"\"\n\n    def __init__(\n        self, in_channels, out_channels, bilinear=True, scale_factor=2, norm=nn.BatchNorm1d\n    ):\n        super().__init__()\n\n        # if bilinear, use the normal convolutions to reduce the number of channels\n        if bilinear:\n            self.up = nn.Upsample(scale_factor=scale_factor, mode=\"bilinear\", align_corners=True)\n            self.conv = DoubleConv(in_channels, out_channels, in_channels // 2, norm=norm)\n        else:\n            self.up = nn.ConvTranspose1d(\n                in_channels, in_channels // 2, kernel_size=scale_factor, stride=scale_factor\n            )\n            self.conv = DoubleConv(in_channels, out_channels, norm=norm)\n\n    def forward(self, x1, x2):\n        x1 = self.up(x1)\n        # input is CHW\n        diff = x2.size()[2] - x1.size()[2]\n        x1 = F.pad(x1, [diff // 2, diff - diff // 2])\n        x = torch.cat([x2, x1], dim=1)\n        return self.conv(x)\n\n\ndef create_layer_norm(channel, length):\n    return nn.LayerNorm([channel, length])\n\n\nclass UNet1DDecoder(nn.Module):\n    def __init__(\n        self,\n        n_channels: int,\n        n_classes: int,\n        duration: int,\n        bilinear: bool = False,\n        se: bool = False,\n        res: bool = False,\n        scale_factor: int = 2,\n        dropout: float = 0.2,\n    ):\n        super().__init__()\n        self.n_channels = n_channels\n        self.n_classes = n_classes\n        self.duration = duration\n        self.bilinear = bilinear\n        self.se = se\n        self.res = res\n        self.scale_factor = scale_factor\n\n        factor = 2 if bilinear else 1\n        self.inc = DoubleConv(\n            self.n_channels, 64, norm=partial(create_layer_norm, length=self.duration)\n        )\n        self.down1 = Down(\n            64, 128, scale_factor, norm=partial(create_layer_norm, length=self.duration // 2)\n        )\n        self.down2 = Down(\n            128, 256, scale_factor, norm=partial(create_layer_norm, length=self.duration // 4)\n        )\n        self.down3 = Down(\n            256, 512, scale_factor, norm=partial(create_layer_norm, length=self.duration // 8)\n        )\n        self.down4 = Down(\n            512,\n            1024 // factor,\n            scale_factor,\n            norm=partial(create_layer_norm, length=self.duration // 16),\n        )\n        self.up1 = Up(\n            1024,\n            512 // factor,\n            bilinear,\n            scale_factor,\n            norm=partial(create_layer_norm, length=self.duration // 8),\n        )\n        self.up2 = Up(\n            512,\n            256 // factor,\n            bilinear,\n            scale_factor,\n            norm=partial(create_layer_norm, length=self.duration // 4),\n        )\n        self.up3 = Up(\n            256,\n            128 // factor,\n            bilinear,\n            scale_factor,\n            norm=partial(create_layer_norm, length=self.duration // 2),\n        )\n        self.up4 = Up(\n            128, 64, bilinear, scale_factor, norm=partial(create_layer_norm, length=self.duration)\n        )\n\n        self.cls = nn.Sequential(\n            nn.Conv1d(64, 64, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.Conv1d(64, self.n_classes, kernel_size=1, padding=0),\n            nn.Dropout(dropout),\n        )\n        self.loss_fn = nn.BCEWithLogitsLoss()\n\n    def forward(\n        self, x: torch.Tensor, labels: Optional[torch.Tensor] = None\n    ) -> dict[str, Optional[torch.Tensor]]:\n        \"\"\"Forward\n\n        Args:\n            x (torch.Tensor): (batch_size, n_channels, n_timesteps)\n\n        Returns:\n            torch.Tensor: (batch_size, n_timesteps, n_classes)\n        \"\"\"\n\n        # 1D U-Net\n        x1 = self.inc(x)\n        x2 = self.down1(x1)\n        x3 = self.down2(x2)\n        x4 = self.down3(x3)\n        x = self.down4(x4)\n        x = self.up1(x, x4)\n        x = self.up2(x, x3)\n        x = self.up3(x, x2)\n        x = self.up4(x, x1)\n\n        # classifier\n        logits = self.cls(x)  # (batch_size, n_classes, n_timesteps)\n        return logits.transpose(1, 2)  # (batch_size, n_timesteps, n_classes)\n\nclass CNNSpectrogram(nn.Module):\n    def __init__(\n        self,\n        in_channels: int = 4,\n        base_filters: int | tuple = 64,\n        kernel_sizes: tuple = (32, 16, 4),\n        stride: int = 4,\n        sigmoid: bool = True,\n        output_size: Optional[int] = None,\n        conv: Callable = nn.Conv1d,\n        reinit: bool = True,\n    ):\n        super().__init__()\n        self.out_chans = len(kernel_sizes)\n        self.out_size = output_size\n        self.sigmoid = sigmoid\n        if isinstance(base_filters, int):\n            base_filters = tuple([base_filters])\n        self.height = base_filters[-1]\n        self.spec_conv = nn.ModuleList()\n        for i in range(self.out_chans):\n            tmp_block = [\n                conv(\n                    in_channels,\n                    base_filters[0],\n                    kernel_size=kernel_sizes[i],\n                    stride=stride,\n                    padding=(kernel_sizes[i] - 1) // 2,\n                )\n            ]\n            if len(base_filters) > 1:\n                for j in range(len(base_filters) - 1):\n                    tmp_block = tmp_block + [\n                        nn.BatchNorm1d(base_filters[j]),\n                        nn.ReLU(inplace=True),\n                        conv(\n                            base_filters[j],\n                            base_filters[j + 1],\n                            kernel_size=kernel_sizes[i],\n                            stride=stride,\n                            padding=(kernel_sizes[i] - 1) // 2,\n                        ),\n                    ]\n                self.spec_conv.append(nn.Sequential(*tmp_block))\n            else:\n                self.spec_conv.append(tmp_block[0])\n\n        if self.out_size is not None:\n            self.pool = nn.AdaptiveAvgPool2d((None, self.out_size))\n\n        if reinit:\n            for m in self.modules():\n                if isinstance(m, nn.Conv1d):\n                    nn.init.kaiming_normal_(m.weight)\n                elif isinstance(m, nn.BatchNorm1d):\n                    nn.init.constant_(m.weight, 1)\n                    nn.init.constant_(m.bias, 0)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        \"\"\"Forward pass of the model.\n\n        Args:\n            x (_type_): (batch_size, in_channels, time_steps)\n\n        Returns:\n            _type_: (batch_size, out_chans, height, time_steps)\n        \"\"\"\n        # x: (batch_size, in_channels, time_steps)\n        out: list[torch.Tensor] = []\n        for i in range(self.out_chans):\n            out.append(self.spec_conv[i](x))\n        img = torch.stack(out, dim=1)  # (batch_size, out_chans, height, time_steps)\n        if self.out_size is not None:\n            img = self.pool(img)  # (batch_size, out_chans, height, out_size)\n        if self.sigmoid:\n            img = img.sigmoid()\n        return img\n    \ndef get_feature_extractor(feature_dim: int, num_timesteps: int\n):\n    feature_extractor = CNNSpectrogram(\n        in_channels=feature_dim, output_size=num_timesteps\n    )\n    return feature_extractor\n\ndef get_decoder(n_channels: int, n_classes: int, num_timesteps: int\n):\n    decoder = UNet1DDecoder(\n        n_channels=n_channels,\n        n_classes=n_classes,\n        duration=num_timesteps\n    )\n\n    return decoder","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:25.518037Z","iopub.execute_input":"2023-12-03T22:22:25.518466Z","iopub.status.idle":"2023-12-03T22:22:25.598080Z","shell.execute_reply.started":"2023-12-03T22:22:25.518423Z","shell.execute_reply":"2023-12-03T22:22:25.597187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model(\n    feature_dim: int,\n    n_classes: int,\n    num_timesteps: int,\n    model_name,\n    test: bool = True,\n    \n) -> BaseModel:\n    model: BaseModel\n    feature_extractor = get_feature_extractor(\n        feature_dim, num_timesteps\n    )\n    decoder = get_decoder(feature_extractor.height, n_classes, num_timesteps)\n    model = Spec2DCNN(\n        feature_extractor=feature_extractor,\n        decoder=decoder,\n        in_channels=feature_extractor.out_chans,\n        mixup_alpha=0.0,\n        cutmix_alpha=0.0,\n        encoder_weights=None,\n        encoder_name=model_name,\n        )\n\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:31:35.882197Z","iopub.execute_input":"2023-12-03T22:31:35.882591Z","iopub.status.idle":"2023-12-03T22:31:35.889536Z","shell.execute_reply.started":"2023-12-03T22:31:35.882558Z","shell.execute_reply":"2023-12-03T22:31:35.888551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\n\nfrom scipy.signal import find_peaks\n\n\ndef post_process_for_seg(\n    keys: list[str], preds: np.ndarray, score_th: float = 0.01, distance: int = 5000\n) -> pl.DataFrame:\n\n    series_ids = np.array(list(map(lambda x: x.split(\"_\")[0], keys)))\n    unique_series_ids = np.unique(series_ids)\n\n    records = []\n    for series_id in unique_series_ids:\n        series_idx = np.where(series_ids == series_id)[0]\n        this_series_preds = preds[series_idx].reshape(-1, 2)\n\n        for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n            this_event_preds = this_series_preds[:, i]\n            steps = find_peaks(this_event_preds, height=score_th, distance=distance)[0]\n            scores = this_event_preds[steps]\n\n            for step, score in zip(steps, scores):\n                records.append(\n                    {\n                        \"series_id\": series_id,\n                        \"step\": step + 0.5,\n                        \"event\": event_name,\n                        \"score\": score,\n                    }\n                )\n\n    if len(records) == 0: \n        records.append(\n            {\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            }\n        )\n\n    sub_df = pl.DataFrame(records).sort(by=[\"series_id\", \"step\"])\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:25.606545Z","iopub.execute_input":"2023-12-03T22:22:25.606800Z","iopub.status.idle":"2023-12-03T22:22:25.618697Z","shell.execute_reply.started":"2023-12-03T22:22:25.606777Z","shell.execute_reply":"2023-12-03T22:22:25.617849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport os\nimport random\nimport sys\nimport time\nfrom contextlib import contextmanager\n\nimport numpy as np\nimport pandas as pd\nimport psutil\n\n\n@contextmanager\ndef trace(title):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info().rss / 2.0**30\n    yield\n    m1 = p.memory_info().rss / 2.0**30\n    delta = m1 - m0\n    sign = \"+\" if delta >= 0 else \"-\"\n    delta = math.fabs(delta)\n    print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} \", file=sys.stderr)\n\n\ndef pad_if_needed(x: np.ndarray, max_len: int, pad_value: float = 0.0) -> np.ndarray:\n    if len(x) == max_len:\n        return x\n    num_pad = max_len - len(x)\n    n_dim = len(x.shape)\n    pad_widths = [(0, num_pad)] + [(0, 0) for _ in range(n_dim - 1)]\n    return np.pad(x, pad_width=pad_widths, mode=\"constant\", constant_values=pad_value)\n\n\ndef nearest_valid_size(input_size: int, downsample_rate: int) -> int:\n    \"\"\"\n    (x // hop_length) % 32 == 0\n    を満たすinput_sizeに最も近いxを返す\n    \"\"\"\n\n    while (input_size // downsample_rate) % 32 != 0:\n        input_size += 1\n    assert (input_size // downsample_rate) % 32 == 0\n\n    return input_size\n\n\ndef random_crop(pos: int, duration: int, max_end) -> tuple[int, int]:\n    \"\"\"Randomly crops with duration length including pos.\n    However, 0<=start, end<=max_end\n    \"\"\"\n    start = random.randint(max(0, pos - duration), min(pos, max_end - duration))\n    end = start + duration\n    return start, end\n\n\ndef negative_sampling(this_event_df: pd.DataFrame, num_steps: int) -> int:\n    \"\"\"negative sampling\n\n    Args:\n        this_event_df (pd.DataFrame): event df\n        num_steps (int): number of steps in this series\n\n    Returns:\n        int: negative sample position\n    \"\"\"\n    # onsetとwakupを除いた範囲からランダムにサンプリング\n    positive_positions = set(this_event_df[[\"onset\", \"wakeup\"]].to_numpy().flatten().tolist())\n    negative_positions = list(set(range(num_steps)) - positive_positions)\n    return random.sample(negative_positions, 1)[0]\n","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:25.622186Z","iopub.execute_input":"2023-12-03T22:22:25.622450Z","iopub.status.idle":"2023-12-03T22:22:25.639344Z","shell.execute_reply.started":"2023-12-03T22:22:25.622426Z","shell.execute_reply":"2023-12-03T22:22:25.638510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import random\n\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nfrom torch.utils.data import Dataset\nfrom torchvision.transforms.functional import resize\n\n\n###################\n# Label\n###################\ndef get_seg_label(\n    this_event_df: pd.DataFrame, num_frames: int, duration: int, start: int, end: int\n) -> np.ndarray:\n    # # (start, end)の範囲と(onset, wakeup)の範囲が重なるものを取得\n    this_event_df = this_event_df.query(\"@start <= wakeup & onset <= @end\")\n\n    label = np.zeros((num_frames, 3))\n    # onset, wakeup, sleepのラベルを作成\n    for onset, wakeup in this_event_df[[\"onset\", \"wakeup\"]].to_numpy():\n        onset = int((onset - start) / duration * num_frames)\n        wakeup = int((wakeup - start) / duration * num_frames)\n        if onset >= 0 and onset < num_frames:\n            label[onset, 1] = 1\n        if wakeup < num_frames and wakeup >= 0:\n            label[wakeup, 2] = 1\n\n        onset = max(0, onset)\n        wakeup = min(num_frames, wakeup)\n        label[onset:wakeup, 0] = 1  # sleep\n\n    return label\n\n\n# ref: https://www.kaggle.com/competitions/dfl-bundesliga-data-shootout/discussion/360236#2004730\ndef gaussian_kernel(length: int, sigma: int = 3) -> np.ndarray:\n    x = np.ogrid[-length : length + 1]\n    h = np.exp(-(x**2) / (2 * sigma * sigma))  # type: ignore\n    h[h < np.finfo(h.dtype).eps * h.max()] = 0\n    return h\n\n\ndef gaussian_label(label: np.ndarray, offset: int, sigma: int) -> np.ndarray:\n    num_events = label.shape[1]\n    for i in range(num_events):\n        label[:, i] = np.convolve(label[:, i], gaussian_kernel(offset, sigma), mode=\"same\")\n\n    return label\n\nclass SegTestDataset(Dataset):\n    def __init__(\n        self,\n        chunk_features: dict[str, np.ndarray],\n    ):\n        self.chunk_features = chunk_features\n        self.keys = list(chunk_features.keys())\n        self.num_features = 4\n        self.upsampled_num_frames = nearest_valid_size(\n            int(5760 * 1), 4\n        )\n\n    def __len__(self):\n        return len(self.keys)\n\n    def __getitem__(self, idx):\n        key = self.keys[idx]\n        feature = self.chunk_features[key]\n        feature = torch.FloatTensor(feature.T).unsqueeze(0)  # (1, num_features, duration)\n        feature = resize(\n            feature,\n            size=[self.num_features, self.upsampled_num_frames],\n            antialias=False,\n        ).squeeze(0)\n\n        return {\n            \"key\": key,\n            \"feature\": feature,  # (num_features, duration)\n        }\n","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:22:25.640540Z","iopub.execute_input":"2023-12-03T22:22:25.640890Z","iopub.status.idle":"2023-12-03T22:22:25.660943Z","shell.execute_reply.started":"2023-12-03T22:22:25.640838Z","shell.execute_reply":"2023-12-03T22:22:25.660028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\n\nimport numpy as np\nimport polars as pl\nimport torch\nfrom pytorch_lightning import seed_everything\nfrom torch.utils.data import DataLoader\nfrom tqdm import tqdm\n\ndef load_chunk_features(\n    duration: int,\n    feature_names: list[str],\n    series_ids: Optional[list[str]],\n    processed_dir: Path,\n    phase: str,\n) -> dict[str, np.ndarray]:\n    features = {}\n\n    if series_ids is None:\n        series_ids = os.listdir(\"/kaggle/working/processed_data\")\n\n    for series_id in series_ids:\n        series_dir = f\"/kaggle/working/processed_data/test/{series_id}/\"\n        this_feature = []\n        for feature_name in feature_names:\n            this_feature.append(np.load(series_dir + f\"{feature_name}.npy\"))\n        this_feature = np.stack(this_feature, axis=1)\n        num_chunks = (len(this_feature) // duration) + 1\n        for i in range(num_chunks):\n            chunk_feature = this_feature[i * duration : (i + 1) * duration]\n            chunk_feature = pad_if_needed(chunk_feature, duration, pad_value=0)  # type: ignore\n            features[f\"{series_id}_{i:07}\"] = chunk_feature\n\n    return features  # type: ignore\n\ndef get_test_ds(\n    chunk_features: dict[str, np.ndarray],\n):\n    return SegTestDataset(chunk_features=chunk_features)\n\ndef pad_if_needed(x: np.ndarray, max_len: int, pad_value: float = 0.0) -> np.ndarray:\n    if len(x) == max_len:\n        return x\n    num_pad = max_len - len(x)\n    n_dim = len(x.shape)\n    pad_widths = [(0, num_pad)] + [(0, 0) for _ in range(n_dim - 1)]\n    return np.pad(x, pad_width=pad_widths, mode=\"constant\", constant_values=pad_value)\n        \nfrom contextlib import contextmanager        \n@contextmanager\ndef trace(title):\n    t0 = time.time()\n    p = psutil.Process(os.getpid())\n    m0 = p.memory_info().rss / 2.0**30\n    yield\n    m1 = p.memory_info().rss / 2.0**30\n    delta = m1 - m0\n    sign = \"+\" if delta >= 0 else \"-\"\n    delta = math.fabs(delta)\n    print(f\"[{m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec] {title} \", file=sys.stderr)\n\ndef nearest_valid_size(input_size: int, downsample_rate: int) -> int:\n    while (input_size // downsample_rate) % 32 != 0:\n        input_size += 1\n    assert (input_size // downsample_rate) % 32 == 0\n\n    return input_size\n\n\ndef load_model(exp_name, model_name):\n    num_timesteps = nearest_valid_size(int(5760 * 1), 4)\n    model = get_model(\n        feature_dim=4,\n        n_classes=3,\n        num_timesteps=num_timesteps // 4,\n        model_name=model_name)\n\n    # load weights\n    weight_path = (\n        \"/kaggle/input/cmi-model/\" + exp_name + \"/single/best_model.pth\"\n    )\n    model.load_state_dict(torch.load(weight_path))\n    print('load weight from \"{}\"'.format(weight_path))\n    return model\n\n\ndef get_test_dataloader():\n    feature_dir = \"/kaggle/working/processed_data/test\"\n    series_ids = os.listdir(feature_dir)\n    #     series_ids = [x.name for x in feature_dir.glob(\"*\")]\n    chunk_features = load_chunk_features(\n        duration=5760,\n        feature_names=[\"anglez\", \"enmo\", \"hour_sin\", \"hour_cos\"],\n        series_ids=series_ids,\n        processed_dir=\"/kaggle/working/processed_data\",\n        phase=\"test\",\n    )\n    test_dataset = get_test_ds(chunk_features=chunk_features)\n    test_dataloader = DataLoader(\n    test_dataset,\n    batch_size=64,\n    shuffle=False,\n    num_workers=2,\n    pin_memory=True,\n    drop_last=False)\n    return test_dataloader\n\n\ndef inference(\n    duration: int, loader: DataLoader, model: BaseModel, device: torch.device, use_amp\n) -> tuple[list[str], np.ndarray]:\n    model = model.to(device)\n    model.eval()\n\n    preds = []\n    keys = []\n    for batch in tqdm(loader, desc=\"inference\"):\n        with torch.no_grad():\n            with torch.cuda.amp.autocast(enabled=use_amp):\n                x = batch[\"feature\"].to(device)\n                output = model.predict(\n                    x,\n                    org_duration=duration,\n                )\n            if output.preds is None:\n                raise ValueError(\"output.preds is None\")\n            else:\n                key = batch[\"key\"]\n                preds.append(output.preds.detach().cpu().numpy())\n                keys.extend(key)\n\n    preds = np.concatenate(preds)\n\n    return keys, preds  # type: ignore\n\n\ndef make_submission(\n    keys: list[str], preds: np.ndarray, score_th, distance\n) -> pl.DataFrame:\n    sub_df = post_process_for_seg(\n        keys,\n        preds,  # type: ignore\n        score_th=score_th,\n        distance=distance,  # type: ignore\n    )\n\n    return sub_df    \n","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:32:37.907872Z","iopub.execute_input":"2023-12-03T22:32:37.908260Z","iopub.status.idle":"2023-12-03T22:32:37.932035Z","shell.execute_reply.started":"2023-12-03T22:32:37.908228Z","shell.execute_reply":"2023-12-03T22:32:37.931218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nseed_everything(42)\n# Load stuff\n\ndef get_preds(exp_name, model_name):\n    with trace(\"load test dataloader\"):\n        test_dataloader = get_test_dataloader()\n    with trace(\"load model\"):\n        model = load_model(exp_name, model_name)\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n\n    with trace(\"inference\"):\n        keys, preds = inference(5760, test_dataloader, model, device, use_amp=True)\n\n    return keys, preds","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:33:43.195173Z","iopub.execute_input":"2023-12-03T22:33:43.195557Z","iopub.status.idle":"2023-12-03T22:33:43.202675Z","shell.execute_reply.started":"2023-12-03T22:33:43.195520Z","shell.execute_reply":"2023-12-03T22:33:43.201769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keys, preds1 = get_preds('submit-55/submit-55', 'resnet34') # 0.723 sub\n_, preds2 = get_preds('submit-bs8', 'resnet34') # 0.726 sub\n_, preds3 = get_preds('submit-bs4', 'resnet34') # 0.725 sub\n_, preds4 = get_preds('excluded', 'resnet50')\n\n# _, preds2 = get_preds('really_long', 'resnet50')\n# _, preds3 = get_preds('resnet18', 'resnet18')\n# _, preds5 = get_preds('second_run', 'resnet34')\n\n# Stack the arrays along a new axis (axis=3) to make it shape (3, 5000, 2, 3)\nstacked_arrays = np.stack([preds1, preds2, preds3, preds4], axis=-1)\n\n# Take the mean along the last axis (axis=-1)\npreds = np.mean(stacked_arrays, axis=-1)\n\nwith trace(\"make submission\"):\n    sub_df = make_submission(\n        keys,\n        preds,\n        score_th=0.005,\n        distance=96,\n    )\nsub_df.write_csv(\"/kaggle/working/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-03T22:39:39.630417Z","iopub.execute_input":"2023-12-03T22:39:39.630836Z","iopub.status.idle":"2023-12-03T22:39:43.161034Z","shell.execute_reply.started":"2023-12-03T22:39:39.630802Z","shell.execute_reply":"2023-12-03T22:39:43.159912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/processed_data\n!rm -rf /kaggle/working/output","metadata":{},"execution_count":null,"outputs":[]}]}