{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6737194,"sourceType":"datasetVersion","datasetId":3809553},{"sourceId":7129650,"sourceType":"datasetVersion","datasetId":3999459}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\nfrom tqdm import tqdm\nfrom collections import defaultdict\nimport math\nfrom typing import Union\n\nimport numpy as np\nimport pandas as pd\n\nimport torch\nfrom torch import nn\nfrom transformers import BertModel, BertTokenizer, Trainer, TrainingArguments\n\nimport matplotlib.pyplot as plt\n\ntolerances = [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nprint(device)\n\ndata_dir = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")\npdata_dir = Path(\"/kaggle/input/cmi-dss-processed/full_data\")\nmodel_dir = Path(\"/kaggle/input/cmi-dss-models\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-14T06:33:57.385654Z","iopub.execute_input":"2023-11-14T06:33:57.385975Z","iopub.status.idle":"2023-11-14T06:34:12.137176Z","shell.execute_reply.started":"2023-11-14T06:33:57.385936Z","shell.execute_reply":"2023-11-14T06:34:12.136218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class StochasticDepth(nn.Module):\n    def __init__(self, prob=0.1):\n        super().__init__()\n        self.prob = prob\n    \n    def forward(self, skip, x):\n        if self.training:\n            return skip if torch.rand(1).item() <= self.prob else (skip + x) / 2\n        return (skip + x) / 2\n\n    \nclass ChannelNorm(nn.Module):\n    def __init__(self, channels=128, norm_type=\"simple\", dim_mask=[0,1,0], eps=1e-6):\n        super().__init__()\n        assert norm_type in (\"simple\", \"learnable\", \"projector\")\n        \n        self.channels = channels\n        self.eps = eps\n        self.dim = [i for i, x in enumerate(dim_mask) if x == 0]\n        \n        def get_simple_stat_func():\n            \n            def simple_stat_func(mu, std):\n                return 1.0, 1.0\n            \n            return simple_stat_func\n        \n        def get_learnable_stat_func():\n            shape = [1 if x == 0 else channels for i, x in enumerate(dim_mask)]\n            self._mu_param = nn.Parameter(torch.rand(shape))\n            self._std_param = nn.Parameter(torch.rand(shape))\n            \n            def learnable_stat_func(mu, std):\n                return self._mu_param, self._std_param\n            \n            return learnable_stat_func\n        \n        def get_projector_stat_func():\n            self._mu_proj = nn.Linear(channels, channels)\n            self._std_proj = nn.Linear(channels, channels)\n            \n            def projector_stat_func(mu, std):\n                go = lambda x, f: f(x.reshape(1, - 1)).reshape(x.shape)\n                return go(mu, self._mu_proj), go(std, self._std_proj)\n            \n            return projector_stat_func\n        \n        self.stat_func = {\n            \"simple\": get_simple_stat_func(), \n            \"learnable\": get_learnable_stat_func(),\n            \"projector\": get_projector_stat_func()\n        }[norm_type]\n    \n    def forward(self, x):\n        mu = x.mean(dim=self.dim, keepdim=True)\n        std = torch.clip(x.std(dim=self.dim, keepdim=True), self.eps)\n        x = (x - mu) / std\n        \n        mu, std = self.stat_func(mu, std)\n        return x * mu + std\n    \n    \nclass TemporalBlock(nn.Module):\n    \"\"\"\n    With little modifications\n    https://github.com/timetoai/TimeDiffusion/blob/main/src/timediffusion/layers.py\n    \"\"\"\n    def __init__(self, chan_in, chan_out, kernel_size, dilation,\n                layers=2, dropout=0.0):\n        super().__init__()\n        \n        net = []\n        for i in range(layers):\n            net.append(nn.Conv1d(\n                (chan_in if i == 0 else chan_out), chan_out, kernel_size,\n                padding=\"same\", dilation=dilation\n            ))\n            net.append(nn.LeakyReLU(0.2))\n            net.append(ChannelNorm(channels=chan_out, norm_type=\"learnable\"))\n        if dropout > 0:\n            net.append(nn.Dropout(dropout))\n        self.net = nn.ModuleList(net)\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        out = x\n        for i in range(len(self.net)):\n            out = self.net[i](out)\n        return out\n    \n\nclass TimeDiffusionProjector(nn.Module):\n    \"\"\"\n    With little modifications\n    https://github.com/timetoai/TimeDiffusion/blob/main/src/timediffusion/models.py\n    \"\"\"\n    def __init__(self, \n                 in_chan: int,\n                 out_chan: int,\n                 blocks: int = 10,\n                 kernel_size: int = 3,\n                 conv_filters: int = 128,\n                 per_block_dropout=0.0,\n                 sd_prob=0.1):\n        super().__init__()\n        self.tcn = nn.ModuleList([\n            TemporalBlock(conv_filters if i > 1 else in_chan, conv_filters, kernel_size,\n                          dilation=i, dropout=per_block_dropout)\n                    for i in [kernel_size ** i for i in range(blocks)]\n                ] + [\n            TemporalBlock(conv_filters, conv_filters, kernel_size,\n                          dilation=i, dropout=per_block_dropout)\n                    for i in [kernel_size ** i for i in range(blocks - 1, - 1, - 1)]\n        ])\n        \n        self.last = nn.Conv1d(conv_filters, out_chan, kernel_size=1)\n        self.sd = StochasticDepth()\n\n    def forward(self, x: torch.Tensor) -> torch.Tensor:\n        skips = []\n        for i in range(len(self.tcn) // 2):\n            x = self.tcn[i](x)\n            skips.append(x)\n        for i in range(len(self.tcn) // 2, len(self.tcn)):\n            x = self.sd(skips[len(self.tcn) // 2 - i - 1], x)\n            x = self.tcn[i](x)\n        x = self.last(x)\n        return nn.functional.silu(x)\n\n    \nclass MinPool1d(nn.MaxPool1d):\n    def __init__(self, kernel_size, stride=None, padding=0, dilation=1, return_indices=False, ceil_mode=False):\n        super().__init__(kernel_size=kernel_size, stride=stride, padding=padding,\n                         dilation=dilation, return_indices=return_indices, ceil_mode=ceil_mode)\n    \n    @torch.no_grad()\n    def forward(self, x):\n        return (- super().forward(- x.float())).long()\n\n    \nclass Model(nn.Module):\n    def __init__(self, feat_dim=16, conv_filters=128, td_dropout=0.02, td_blocks=10,\n                 classes=2, classif_layers=3):\n        super().__init__()\n\n        self.enmo_anglez_proj = nn.Linear(2 * 12, feat_dim)\n        self.min_emb = nn.Sequential(MinPool1d(kernel_size=12, ceil_mode=True),\n                                     nn.Embedding(60, feat_dim))\n        self.hour_emb = nn.Sequential(MinPool1d(kernel_size=12, ceil_mode=True),\n                                     nn.Embedding(24, feat_dim))\n        self.weekday_emb = nn.Sequential(MinPool1d(kernel_size=12, ceil_mode=True),\n                                     nn.Embedding(7, feat_dim))\n        self.month_emb = nn.Sequential(MinPool1d(kernel_size=12, ceil_mode=True),\n                                     nn.Embedding(12, feat_dim))\n        \n#         self.enc = nn.TransformerEncoder(\n#             nn.TransformerEncoderLayer(feat_dim * 5, 1, feat_dim * 5,\n#                                        batch_first=True),\n#                                 num_layers=1)\n        \n        out_size = feat_dim * 4\n        self.tdp = TimeDiffusionProjector(in_chan=feat_dim * 5, out_chan=out_size,\n                                                      kernel_size=3, blocks=td_blocks,\n                                                      conv_filters=conv_filters,\n                                                      per_block_dropout=td_dropout)\n        \n        self.classif_heads = []\n        out_size_sqr = int(out_size ** 0.5)\n        for tol in tolerances:\n            tol = tol // 12\n            if tol % 2 == 0: tol += 1\n            self.classif_heads.append([\n                nn.Conv1d(out_size, out_size_sqr, kernel_size=1, padding=\"same\"),\n                nn.LeakyReLU(0.2),\n                ChannelNorm(channels=out_size_sqr, norm_type=\"learnable\"),\n                \n                nn.Conv1d(out_size_sqr, 1, kernel_size=tol, padding=\"same\"),\n                nn.Sigmoid(),\n                nn.AvgPool1d(kernel_size=tol, stride=1, padding=tol // 2),\n            ])\n        self.classif_heads = nn.ModuleList([nn.Sequential(*x) for x in self.classif_heads])\n\n    def forward(self, x):\n        if len(x[0].shape) == 2:\n            x = [f.unsqueeze(0) for f in x]\n            \n        enmo_anglez, minutes, hours, weekday, month = x\n        if enmo_anglez.shape[- 2] % 12 > 0:\n            enmo_anglez = nn.functional.pad(enmo_anglez, (0, 0, 0, 12 - (enmo_anglez.shape[- 2] % 12)), \"replicate\")\n        enmo_anglez = enmo_anglez.reshape(enmo_anglez.shape[0], enmo_anglez.shape[1] // 12, 2 * 12)\n        x = torch.cat([\n            self.enmo_anglez_proj(enmo_anglez.to(device)),\n            self.min_emb(minutes.to(device)),\n            self.hour_emb(hours.to(device)),\n            self.weekday_emb(weekday.to(device)),\n            self.month_emb(month.to(device))\n        ], axis=-1)\n        \n#         x = self.enc(x)\n        x = self.tdp(torch.transpose(x, - 1, - 2))\n        \n        x = torch.mean(torch.stack([head(x) for head in self.classif_heads], 0), axis=0)\n        x = torch.transpose(x, - 1, - 2)\n        x = x.repeat_interleave(12, dim=- 2)[..., :minutes.shape[- 1], :]\n        return x.squeeze(2)","metadata":{"execution":{"iopub.status.busy":"2023-11-14T06:34:12.138944Z","iopub.execute_input":"2023-11-14T06:34:12.139544Z","iopub.status.idle":"2023-11-14T06:34:12.150862Z","shell.execute_reply.started":"2023-11-14T06:34:12.139516Z","shell.execute_reply":"2023-11-14T06:34:12.149983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_config = dict(feat_dim=32, \n                    conv_filters=128, \n                    td_dropout=0.02,\n                    td_blocks=12,\n                    classes=1, \n                    classif_layers=2)\n\nmodel = Model(**model_config).to(device, dtype=torch.float32)\nmodel.load_state_dict(torch.load(model_dir / f\"ner_v95_best_model.pth\", map_location=device))\n    \n@torch.no_grad()\ndef predict(data):\n    model.eval()\n    out = model(data).squeeze().detach().cpu().numpy()\n    out_rev = model([x.flip([0]) for x in data]).squeeze().flip([0]).detach().cpu().numpy()\n    return (out + out_rev) / 2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_df(df, is_test=False):\n    feats = torch.from_numpy(df[[\"enmo\", \"anglez\"]].values.astype(np.float32))\n    \n    date = pd.to_datetime(df.timestamp.map(lambda s: s[: - 5]), format=\"%Y-%m-%dT%H:%M:%S\")\n    minutes = torch.from_numpy(date.dt.minute.values)\n    hours = torch.from_numpy(date.dt.hour.values)\n    weekday = torch.from_numpy(date.dt.weekday.values)\n    month = torch.from_numpy(date.dt.month.values - 1)\n    \n    if not is_test:\n        label = torch.from_numpy(df.awake.values.astype(np.float32))\n    else:\n        label = None\n    return (feats, minutes, hours, weekday, month), label","metadata":{"execution":{"iopub.status.busy":"2023-11-14T06:34:19.649479Z","iopub.execute_input":"2023-11-14T06:34:19.649756Z","iopub.status.idle":"2023-11-14T06:34:19.656030Z","shell.execute_reply.started":"2023-11-14T06:34:19.649731Z","shell.execute_reply":"2023-11-14T06:34:19.655179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# test","metadata":{}},{"cell_type":"code","source":"test = pd.read_parquet(data_dir / \"test_series.parquet\")\ntest","metadata":{"execution":{"iopub.status.busy":"2023-10-10T14:38:27.190675Z","iopub.execute_input":"2023-10-10T14:38:27.191045Z","iopub.status.idle":"2023-10-10T14:38:27.214157Z","shell.execute_reply.started":"2023-10-10T14:38:27.191019Z","shell.execute_reply":"2023-10-10T14:38:27.212826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nseq_len = 720 * 24 * 14\n\nfor ts_id in tqdm(test.series_id.unique()):\n    cur = test[test.series_id == ts_id]\n    preds.append(np.zeros(len(cur)))\n    \n    for i in range(0, len(cur), seq_len):\n        data = process_df(cur[i: i + seq_len], is_test=True)[0]\n        preds[- 1][i: i + seq_len] = predict(data)\n    \n    if seq_len < len(cur):\n        data = process_df(cur[- seq_len:], is_test=True)[0]\n        preds[- 1][- seq_len:] = predict(data)\n    \npreds = np.concatenate(preds)\npreds.shape","metadata":{"execution":{"iopub.status.busy":"2023-10-10T14:38:10.889512Z","iopub.execute_input":"2023-10-10T14:38:10.892061Z","iopub.status.idle":"2023-10-10T14:38:15.604684Z","shell.execute_reply.started":"2023-10-10T14:38:10.892023Z","shell.execute_reply":"2023-10-10T14:38:15.603661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def mean_th(arr, window, threshold):\n    assert window % 2 == 1\n    \n    viewf = np.lib.stride_tricks.sliding_window_view(arr, window_shape=(window,))\n    meanf = viewf.mean(axis=1)\n    \n    res = np.empty_like(arr)\n    \n    half = window // 2\n    res[half: - half] = meanf\n    cur = 0\n    for i in range(half):\n        cur += arr[i]\n        res[i] = cur / (i + 1)\n    cur = 0\n    for i in range(len(arr) - 1, len(arr) - half - 1, - 1):\n        cur += arr[i]\n        res[i] = cur / (len(arr) - i)\n        \n    return res >= threshold\n\ndef filter_preds(preds, min_window=12 * 30):\n    i = 0\n    lst = None\n    while i < len(preds):\n        if preds[i] == 1 and lst is not None:\n            if (i - lst + 1) < min_window:\n                preds[lst: i] = 1\n            lst = None\n        elif preds[i] == 0 and lst is None:\n            lst = i\n        i += 1\n    if preds[- 1] == 0 and lst is not None:\n        if (len(preds) - lst) < min_window:\n            preds[lst:] = 1\n    return preds\n\ndef simple_th(arr, threshold):\n    return arr > threshold","metadata":{"execution":{"iopub.status.busy":"2023-10-10T14:38:15.606026Z","iopub.execute_input":"2023-10-10T14:38:15.606945Z","iopub.status.idle":"2023-10-10T14:38:15.617360Z","shell.execute_reply.started":"2023-10-10T14:38:15.606911Z","shell.execute_reply":"2023-10-10T14:38:15.615930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preds_processed = mean_th(preds, window=30 * 12 + 1, threshold=0.54)\nthreshold = 0.50\npreds_processed = simple_th(preds, threshold)\npreds_processed = filter_preds(preds_processed, min_window=12 * 60 * 2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res = []\ncur_ts = test.series_id[0]\ncur = preds_processed[0]\nfor i in tqdm(range(1, len(test))):\n    if test.series_id[i] != cur_ts:\n        cur_ts = test.series_id[i]\n    elif preds_processed[i] != cur:\n        res.append((cur_ts, test.step[i], \"wakeup\" if cur == 0 else \"onset\", preds[i] if cur == 0 else 1 - preds[i]))\n    cur = preds_processed[i]\n    \nsubmission = pd.DataFrame(res, columns=[\"series_id\", \"step\", \"event\", \"score\"])\nsubmission[\"row_id\"] = submission.index\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-10-10T14:38:15.622003Z","iopub.execute_input":"2023-10-10T14:38:15.622565Z","iopub.status.idle":"2023-10-10T14:38:15.815718Z","shell.execute_reply.started":"2023-10-10T14:38:15.622503Z","shell.execute_reply":"2023-10-10T14:38:15.814427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-10-10T14:38:15.816973Z","iopub.status.idle":"2023-10-10T14:38:15.817697Z","shell.execute_reply.started":"2023-10-10T14:38:15.817451Z","shell.execute_reply":"2023-10-10T14:38:15.817475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}