{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":154116255,"sourceType":"kernelVersion"}],"dockerImageVersionId":30588,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nfrom tqdm.auto import tqdm\nfrom scipy.signal import find_peaks\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-08T08:44:11.009565Z","iopub.execute_input":"2023-12-08T08:44:11.010520Z","iopub.status.idle":"2023-12-08T08:44:11.016603Z","shell.execute_reply.started":"2023-12-08T08:44:11.010480Z","shell.execute_reply":"2023-12-08T08:44:11.015366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\nMODEL_DIR = \"/kaggle/input/cmi-14th-place-solution-train/\"\nPOSTPROCESS = True","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:11.018364Z","iopub.execute_input":"2023-12-08T08:44:11.018949Z","iopub.status.idle":"2023-12-08T08:44:11.038850Z","shell.execute_reply.started":"2023-12-08T08:44:11.018916Z","shell.execute_reply":"2023-12-08T08:44:11.037644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Data Preparation","metadata":{}},{"cell_type":"code","source":"df = pl.read_parquet(INPUT_DIR + \"train_series.parquet\", n_rows=200000)\ndf = (\n    df.with_columns(\n        pl.col(\"timestamp\").str.to_datetime(),\n    ).with_columns(\n        pl.col(\"timestamp\").dt.date().cast(str).alias(\"date\"),\n        pl.col(\"timestamp\").dt.time().cast(str).alias(\"time\"),\n    )\n).to_pandas()\n\ndf_dummy_1 = pd.DataFrame(columns=sorted(df[\"time\"].unique()), dtype=\"float32\")\ndisplay(df_dummy_1)\n\ndf_dummy_2 = pd.read_parquet(MODEL_DIR + \"df_mask.parquet\").iloc[:0]\ndisplay(df_dummy_2)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:11.041097Z","iopub.execute_input":"2023-12-08T08:44:11.041927Z","iopub.status.idle":"2023-12-08T08:44:12.724008Z","shell.execute_reply.started":"2023-12-08T08:44:11.041890Z","shell.execute_reply":"2023-12-08T08:44:12.722444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_series = pl.read_parquet(INPUT_DIR + \"test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:12.725575Z","iopub.execute_input":"2023-12-08T08:44:12.726026Z","iopub.status.idle":"2023-12-08T08:44:12.735471Z","shell.execute_reply.started":"2023-12-08T08:44:12.725994Z","shell.execute_reply":"2023-12-08T08:44:12.733902Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_groupby = df_series.group_by(\"series_id\", maintain_order=True)\n\ndict_valid_ratio = dict()\nlist_feature_array = []\nlist_df_1min = []\nfor series_id, df in tqdm(df_groupby, total=df_series.get_column(\"series_id\").n_unique()):\n    df = (\n        df.with_columns(\n            pl.col(\"timestamp\").str.to_datetime(),\n        ).with_columns(\n            pl.col(\"timestamp\").dt.date().cast(str).alias(\"date\"),\n            pl.col(\"timestamp\").dt.time().cast(str).alias(\"time\"),\n        )\n    ).to_pandas()\n    \n    df[\"timestamp\"] = df[\"timestamp\"].dt.tz_localize(None)\n    dup_count = df.groupby([\"anglez\", \"enmo\", \"time\"])[\"step\"].transform(\"count\")\n    df[\"valid_flag\"] = (dup_count == 1).astype(\"float32\")\n    dict_valid_ratio[series_id] = df[\"valid_flag\"].mean()\n    \n    list_feature_array_tmp = []\n    df[\"log_anglez_std\"] = np.log(df[\"anglez\"].rolling(25, min_periods=1, center=True).std() + 1).astype(\"float32\")\n    df[\"log_enmo\"] = np.log(df[\"enmo\"] + 0.01).astype(\"float32\")\n    for feature in [\"log_anglez_std\", \"log_enmo\", \"valid_flag\"]:\n        df_pivot = df.pivot(index=[\"series_id\", \"date\"], columns=\"time\", values=feature)\n        if df_pivot.shape[1] != df_dummy_1.shape[1]:\n            df_pivot = pd.concat([df_dummy_1, df_pivot])\n        feature_array = df_pivot.fillna(0).values\n        feature_array_1day_bedore = df_pivot.shift(1).fillna(0).values\n        feature_array_1day_after = df_pivot.shift(-1).fillna(0).values\n        feature_array = np.concatenate([feature_array_1day_bedore[:, -180*12:], feature_array, feature_array_1day_after[:, :180*12]], axis=1)\n        list_feature_array_tmp.append(feature_array)\n    list_feature_array.append(np.stack(list_feature_array_tmp, axis=1))\n    \n    dict_agg = {\"series_id\": \"first\", \"date\": \"first\", \"time\": \"first\", \"step\": \"mean\", \"valid_flag\": \"max\"}\n    df_1min = df.resample(\"1min\", on=\"timestamp\").agg(dict_agg).reset_index()\n    df_1min[\"step\"] = df_1min[\"step\"].astype(\"int32\")\n    list_df_1min.append(df_1min)\n\ndel df_series, df_groupby","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:12.739205Z","iopub.execute_input":"2023-12-08T08:44:12.739703Z","iopub.status.idle":"2023-12-08T08:44:12.983698Z","shell.execute_reply.started":"2023-12-08T08:44:12.739657Z","shell.execute_reply":"2023-12-08T08:44:12.982367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.concatenate(list_feature_array)\ndel list_feature_array\nX = (X - X.min(axis=(0, 2), keepdims=True)) / (X.max(axis=(0, 2), keepdims=True) - X.min(axis=(0, 2), keepdims=True))\n\nX.shape","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:12.985471Z","iopub.execute_input":"2023-12-08T08:44:12.985925Z","iopub.status.idle":"2023-12-08T08:44:12.998475Z","shell.execute_reply.started":"2023-12-08T08:44:12.985880Z","shell.execute_reply":"2023-12-08T08:44:12.997008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_1min = pd.concat(list_df_1min)\ndel list_df_1min\n\ndf_1min","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.000833Z","iopub.execute_input":"2023-12-08T08:44:13.001308Z","iopub.status.idle":"2023-12-08T08:44:13.038104Z","shell.execute_reply.started":"2023-12-08T08:44:13.001265Z","shell.execute_reply":"2023-12-08T08:44:13.036621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.040118Z","iopub.execute_input":"2023-12-08T08:44:13.041029Z","iopub.status.idle":"2023-12-08T08:44:13.382699Z","shell.execute_reply.started":"2023-12-08T08:44:13.040977Z","shell.execute_reply":"2023-12-08T08:44:13.381362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mask = df_1min.pivot(index=[\"series_id\", \"date\"], columns=\"time\", values=\"valid_flag\").fillna(0)\ndf_mask.columns","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.384290Z","iopub.execute_input":"2023-12-08T08:44:13.385398Z","iopub.status.idle":"2023-12-08T08:44:13.407704Z","shell.execute_reply.started":"2023-12-08T08:44:13.385350Z","shell.execute_reply":"2023-12-08T08:44:13.406299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_mask = df_1min.pivot(index=[\"series_id\", \"date\"], columns=\"time\", values=\"valid_flag\").fillna(0)\nif df_mask.shape[1] != df_dummy_2.shape[1]:\n    df_mask = pd.concat([df_dummy_2, df_mask]).fillna(0)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.409710Z","iopub.execute_input":"2023-12-08T08:44:13.410209Z","iopub.status.idle":"2023-12-08T08:44:13.433198Z","shell.execute_reply.started":"2023-12-08T08:44:13.410163Z","shell.execute_reply":"2023-12-08T08:44:13.432110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Regression Model (1D-CNN UNet)","metadata":{}},{"cell_type":"markdown","source":"The architecture of my NN model is based on the following notebook written by [@K_mat](https://www.kaggle.com/kmat2019) in the other competition.  \nhttps://www.kaggle.com/code/kmat2019/u-net-1d-cnn-with-keras\n\nThanks and congrats on 2nd Place!","metadata":{}},{"cell_type":"markdown","source":"### 2.1. definition of classes and functions","metadata":{}},{"cell_type":"code","source":"class MyDataset(Dataset):\n    def __init__(self, X, Y, flag):\n        self.X = torch.FloatTensor(X)\n        if Y is not None:\n            self.Y = torch.FloatTensor(Y)\n        self.flag = torch.FloatTensor(flag)\n    \n    def __len__(self):\n        return self.X.shape[0]\n    \n    def __getitem__(self, idx):\n        if \"Y\" in dir(self):\n            return (self.X[idx], self.Y[idx], self.flag[idx])\n        else:\n            return (self.X[idx], torch.Tensor(), self.flag[idx])","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.437423Z","iopub.execute_input":"2023-12-08T08:44:13.439571Z","iopub.status.idle":"2023-12-08T08:44:13.454185Z","shell.execute_reply.started":"2023-12-08T08:44:13.439518Z","shell.execute_reply":"2023-12-08T08:44:13.452973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ConvBNReLU(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, stride, groups=1):\n        super().__init__()\n        \n        if stride == 1:\n            padding = \"same\"\n        else:\n            padding = (kernel_size - stride) // 2\n        self.layers = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=groups),\n            nn.BatchNorm1d(out_channels),\n            nn.ReLU()\n        )\n    \n    def forward(self, x):\n        x_out = self.layers(x)\n        return x_out\n\n\nclass SEBlock(nn.Module):\n    def __init__(self, n_channels, se_ratio):\n        super().__init__()\n        \n        self.layers = nn.Sequential(\n            nn.AdaptiveAvgPool1d(output_size=1),  #  Global Average Pooling\n            nn.Conv1d(n_channels, n_channels//se_ratio, kernel_size=1),\n            nn.ReLU(),\n            nn.Conv1d(n_channels//se_ratio, n_channels, kernel_size=1),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        x_out = torch.mul(x, self.layers(x))\n        return x_out\n\n\nclass ResBlock(nn.Module):\n    def __init__(self, n_channels, kernel_size, se_ratio):\n        super().__init__()\n        \n        self.layers = nn.Sequential(\n            ConvBNReLU(n_channels, n_channels, kernel_size, stride=1),\n            ConvBNReLU(n_channels, n_channels, kernel_size, stride=1),\n            SEBlock(n_channels, se_ratio)\n        )\n    \n    def forward(self, x):\n        x_re = self.layers(x)\n        x_out = x + x_re\n        return x_out\n    \n\nclass UNet1d(nn.Module):\n    def __init__(self, input_channels, initial_channels, initial_kernel_size,\n                 down_channels, down_kernel_size, down_stride, res_depth, res_kernel_size, se_ratio, out_kernel_size):\n        super().__init__()\n        self.down_kernel_size = down_kernel_size\n        self.down_stride = down_stride\n        \n        self.initial_layers = ConvBNReLU(input_channels, initial_channels, initial_kernel_size, stride=1, groups=input_channels)\n        \n        self.down_layers = nn.ModuleList()\n        for i in range(len(down_channels)):\n            if i == 0:\n                in_channels = initial_channels\n            else:\n                in_channels = down_channels[i-1] + input_channels\n            out_channels = down_channels[i]\n            kernel_size = down_kernel_size[i]\n            stride = down_stride[i]\n            \n            block = []\n            block.append(ConvBNReLU(in_channels, out_channels, kernel_size, stride))\n            for j in range(res_depth):\n                block.append(ResBlock(out_channels, res_kernel_size, se_ratio))\n            self.down_layers.append(nn.Sequential(*block))\n        \n        self.up_layers = nn.ModuleList()\n        for i in range(len(down_channels)-1, 0, -1):\n            in_channels = out_channels + down_channels[i]\n            out_channels = down_channels[i]\n            kernel_size = down_kernel_size[i]\n            self.up_layers.append(ConvBNReLU(in_channels, out_channels, kernel_size, stride=1))\n        \n        self.out_layers = nn.Conv1d(down_channels[1], 1, out_kernel_size, padding=\"same\")\n    \n    def forward(self, x):\n        outs = []\n        x_avg = x\n        x = self.initial_layers(x)\n        \n        for i in range(len(self.down_layers)):\n            x_out = self.down_layers[i](x)\n            if i == len(self.down_layers) - 1:\n                x = x_out\n            else:\n                outs.append(x_out)\n                kernel_size = self.down_kernel_size[i]\n                stride = self.down_stride[i]\n                padding = (kernel_size - stride) // 2\n                x_avg = F.avg_pool1d(x_avg, kernel_size, stride, padding)\n                x = torch.cat([x_out, x_avg], dim=1)\n        \n        for i in range(len(self.up_layers)):\n            scale_factor = self.down_stride[-i-1]\n            x = F.interpolate(x, scale_factor=scale_factor, mode=\"linear\")\n            x = torch.cat([x, outs[-i-1]], dim=1)\n            x = self.up_layers[i](x)\n        \n        x_out = self.out_layers(x)\n        x_out = x_out[:, 0, 180:-180]\n        \n        return x_out","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.456447Z","iopub.execute_input":"2023-12-08T08:44:13.457508Z","iopub.status.idle":"2023-12-08T08:44:13.487943Z","shell.execute_reply.started":"2023-12-08T08:44:13.457460Z","shell.execute_reply":"2023-12-08T08:44:13.486832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, data_loader, device):\n    model.eval()\n    \n    preds_all = []\n    for batch in data_loader:\n        X = batch[0].to(device)\n        mask = batch[2].to(device)\n        \n        with torch.no_grad():\n            preds = model(X) * mask\n        preds = preds.cpu().numpy()\n        preds_all.append(preds)\n    \n    preds_all = np.concatenate(preds_all)\n        \n    return preds_all","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.490025Z","iopub.execute_input":"2023-12-08T08:44:13.490706Z","iopub.status.idle":"2023-12-08T08:44:13.507743Z","shell.execute_reply.started":"2023-12-08T08:44:13.490655Z","shell.execute_reply":"2023-12-08T08:44:13.506421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2. inference","metadata":{}},{"cell_type":"code","source":"n_splits = 10\nbatch_size = 16\n\nmask = df_mask.values\npreds_test = np.zeros_like(mask)\n\n# dataset\nds_test = MyDataset(X, None, mask)\n\n# dataloader\ndl_test = DataLoader(ds_test, batch_size=batch_size, shuffle=False, \n                    num_workers=0, pin_memory=True, drop_last=False)\n\n# build model\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel = UNet1d(\n    input_channels=X.shape[1],\n    initial_channels=72,\n    initial_kernel_size=15,\n    down_channels=(72, 72, 72),\n    down_kernel_size=(12, 15, 15),\n    down_stride=(12, 9, 5),  # first element must be 12\n    res_depth=3,\n    res_kernel_size=15,\n    se_ratio=4,\n    out_kernel_size=21,\n)\nmodel.to(device)\n\nfor i in range(n_splits):\n    with torch.no_grad():\n        model.load_state_dict(torch.load(MODEL_DIR + f\"model_{i}.pth\", map_location=device))\n        preds_test += predict(model, dl_test, device) / n_splits","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:13.509836Z","iopub.execute_input":"2023-12-08T08:44:13.511417Z","iopub.status.idle":"2023-12-08T08:44:16.305997Z","shell.execute_reply.started":"2023-12-08T08:44:13.511380Z","shell.execute_reply":"2023-12-08T08:44:16.304669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred = pd.DataFrame(preds_test, index=df_mask.index, columns=df_mask.columns)\ndf_pred = df_pred.stack().reset_index(name=\"score\")\ndf_pred = pd.merge(\n    df_1min[[\"series_id\", \"date\", \"time\", \"step\"]],\n    df_pred,\n    on=[\"series_id\", \"date\", \"time\"],\n    how=\"inner\"\n)\ndf_pred","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:16.307770Z","iopub.execute_input":"2023-12-08T08:44:16.308310Z","iopub.status.idle":"2023-12-08T08:44:16.348405Z","shell.execute_reply.started":"2023-12-08T08:44:16.308260Z","shell.execute_reply":"2023-12-08T08:44:16.346979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Make Submission","metadata":{}},{"cell_type":"markdown","source":"### 3.1. without post-prosessing (only peak detection)","metadata":{}},{"cell_type":"code","source":"if not POSTPROCESS:\n    list_df = []\n    for series_id, df in tqdm(df_pred.groupby(\"series_id\")):\n        for event in [\"onset\", \"wakeup\"]:\n            values_step = df[\"step\"].values\n            if event == \"onset\":\n                values_score = -df[\"score\"].values\n            else:\n                values_score = df[\"score\"].values\n\n            peak_idx = find_peaks(values_score, height=0.0, distance=8)[0]\n            df_peak = pd.DataFrame(values_step[peak_idx], columns=[\"step\"])\n            df_peak[\"series_id\"] = series_id\n            df_peak[\"event\"] = event\n            df_peak[\"score\"] = values_score[peak_idx]\n            list_df.append(df_peak)\n\n    df_sub = pd.concat(list_df)\n    df_sub = df_sub.sort_values(\"score\", ascending=False).groupby(\"event\").head(100000)  # avoid Submission Scoring Error\n    df_sub = df_sub.sort_values([\"series_id\", \"step\"]).reset_index(drop=True)\n    df_sub = df_sub[[\"series_id\", \"step\", \"event\", \"score\"]].reset_index(names=\"row_id\")\n    df_sub.to_csv(\"submission.csv\", index=False)\n    display(df_sub)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:16.349984Z","iopub.execute_input":"2023-12-08T08:44:16.351167Z","iopub.status.idle":"2023-12-08T08:44:16.363954Z","shell.execute_reply.started":"2023-12-08T08:44:16.351099Z","shell.execute_reply":"2023-12-08T08:44:16.362640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.2 with post-processing","metadata":{}},{"cell_type":"code","source":"if POSTPROCESS:\n    df_events = pd.read_csv(INPUT_DIR + \"train_events.csv\").dropna()\n    df_events[\"timestamp\"] = pd.to_datetime(df_events[\"timestamp\"], utc=True).dt.tz_localize(None)\n    df_events[\"time\"] = df_events[\"timestamp\"].dt.time.astype(str)\n    df_events[\"minute_mod15\"] = df_events[\"timestamp\"].dt.minute % 15\n\n    df_agg = df_events.groupby([\"time\", \"event\"], as_index=False).size()\n    df_agg[\"rate\"] = df_agg[\"size\"] / df_agg.groupby(\"event\")[\"size\"].transform(\"sum\") * (60*24)\n    df_time = df_agg.pivot(index=\"time\", columns=\"event\", values=\"rate\").fillna(0).reset_index()\n    df_time = df_time.merge(df_pred[[\"time\"]].drop_duplicates(), how=\"right\").fillna(0)\n    df_time = pd.concat([df_time]*3, ignore_index=True)\n    df_time[\"onset\"] = df_time[\"onset\"].rolling(60, center=True).mean()\n    df_time[\"wakeup\"] = df_time[\"wakeup\"].rolling(60, center=True).mean()\n    df_time = df_time.iloc[60*24:-60*24].reset_index(drop=True)\n\n    df_agg = df_events.groupby([\"minute_mod15\", \"event\"], as_index=False).size()\n    df_agg[\"rate\"] = df_agg[\"size\"] / df_agg.groupby(\"event\")[\"size\"].transform(\"sum\") * 15\n    df_minute = df_agg.pivot(index=\"minute_mod15\", columns=\"event\", values=\"rate\").reset_index()\n    \n    df_time[[\"onset\", \"wakeup\"]] = df_time[[\"onset\", \"wakeup\"]].clip(0.1, 1.1) ** 0.13\n    df_minute[[\"onset\", \"wakeup\"]] = df_minute[[\"onset\", \"wakeup\"]].clip(0.5, 1.3) ** 0.06\n    \n    df_pred[\"minute_mod15\"] = df_pred[\"time\"].str[3:5].astype(int) % 15\n\n    list_df = []\n    for series_id, df in tqdm(df_pred.groupby(\"series_id\")):\n        df = df.merge(df_time, how=\"left\", on=\"time\")\n        df = df.merge(df_minute, how=\"left\", on=\"minute_mod15\")\n\n        df_tmp = df.copy()\n        df_tmp[\"score\"] = df_tmp[\"score\"].replace(0.0, np.nan)\n        df_tmp = df_tmp.groupby(\"time\")[\"score\"].mean()\n        df_tmp = pd.concat([df_tmp]*3).rolling(90, center=True, min_periods=1).mean()\n        df_tmp = df_tmp.iloc[60*24:-60*24].reset_index().rename({\"score\": \"score_mean\"}, axis=1)\n        df = df.merge(df_tmp, on=\"time\", how=\"left\")\n\n        df[\"score\"] = 0.9*df[\"score\"] + 0.1*df[\"score_mean\"]\n        df[\"score\"] *= np.where(df[\"score\"]>0, df[\"wakeup_x\"], df[\"onset_x\"])\n        df[\"score\"] *= np.where(df[\"score\"]>0, df[\"wakeup_y\"], df[\"onset_y\"])\n        valid_ratio = dict_valid_ratio[series_id]\n\n        for event in [\"onset\", \"wakeup\"]:\n            values_step = df[\"step\"].values\n            if event == \"onset\":\n                values_score = -df[\"score\"].values\n            else:\n                values_score = df[\"score\"].values\n\n            # measure peaks\n            peak_idx = find_peaks(values_score, height=0.04, distance=60*16)[0]  # at least 16 hours interval\n            df_measure_peak = pd.DataFrame(values_step[peak_idx], columns=[\"step\"])\n            df_measure_peak[\"series_id\"] = series_id\n            df_measure_peak[\"event\"] = event\n            df_measure_peak[\"score\"] = values_score[peak_idx] * 4 * valid_ratio**0.15\n\n            # minor peaks\n            peak_idx = find_peaks(values_score, height=0.0, distance=6)[0]\n            df_minor_peak = pd.DataFrame(values_step[peak_idx], columns=[\"step\"])\n            df_minor_peak[\"series_id\"] = series_id\n            df_minor_peak[\"event\"] = event\n            df_minor_peak[\"score\"] = values_score[peak_idx]\n\n            df_peak = pd.concat([df_measure_peak, df_minor_peak]).drop_duplicates(subset=[\"step\"])\n            list_df.append(df_peak)\n\n    df_sub = pd.concat(list_df)\n    df_sub = df_sub.sort_values(\"score\", ascending=False).groupby(\"event\").head(100000)  # avoid Submission Scoring Error\n    df_sub = df_sub.sort_values([\"series_id\", \"step\"]).reset_index(drop=True)\n    df_sub = df_sub[[\"series_id\", \"step\", \"event\", \"score\"]].reset_index(names=\"row_id\")\n    df_sub.to_csv(\"submission.csv\", index=False)\n    display(df_sub)","metadata":{"execution":{"iopub.status.busy":"2023-12-08T08:44:16.366306Z","iopub.execute_input":"2023-12-08T08:44:16.367165Z","iopub.status.idle":"2023-12-08T08:44:16.685330Z","shell.execute_reply.started":"2023-12-08T08:44:16.367105Z","shell.execute_reply":"2023-12-08T08:44:16.684014Z"},"trusted":true},"execution_count":null,"outputs":[]}]}