{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6879418,"sourceType":"datasetVersion","datasetId":3785250},{"sourceId":6955368,"sourceType":"datasetVersion","datasetId":3993054},{"sourceId":7027622,"sourceType":"datasetVersion","datasetId":4041874},{"sourceId":7058618,"sourceType":"datasetVersion","datasetId":4063315},{"sourceId":7072803,"sourceType":"datasetVersion","datasetId":3983955},{"sourceId":7076676,"sourceType":"datasetVersion","datasetId":4076187},{"sourceId":7083477,"sourceType":"datasetVersion","datasetId":4080830},{"sourceId":7110296,"sourceType":"datasetVersion","datasetId":4099650},{"sourceId":7116882,"sourceType":"datasetVersion","datasetId":4104401},{"sourceId":7124391,"sourceType":"datasetVersion","datasetId":4109703},{"sourceId":7126599,"sourceType":"datasetVersion","datasetId":4111140},{"sourceId":7130911,"sourceType":"datasetVersion","datasetId":4108642},{"sourceId":144358840,"sourceType":"kernelVersion"},{"sourceId":149254949,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install additional libraries","metadata":{}},{"cell_type":"code","source":"!pip install hydra-core segmentation_models_pytorch==0.3.3 --no-index --find-links=/kaggle/input/ex-library","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:17.886447Z","iopub.execute_input":"2023-12-05T15:47:17.886746Z","iopub.status.idle":"2023-12-05T15:47:39.703829Z","shell.execute_reply.started":"2023-12-05T15:47:17.886719Z","shell.execute_reply":"2023-12-05T15:47:39.702631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport polars as pl\nimport glob\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:39.705676Z","iopub.execute_input":"2023-12-05T15:47:39.705964Z","iopub.status.idle":"2023-12-05T15:47:39.944037Z","shell.execute_reply.started":"2023-12-05T15:47:39.705937Z","shell.execute_reply":"2023-12-05T15:47:39.943184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pred_df(\n    keys: list[str], preds: np.ndarray\n)-> pl.DataFrame:\n    \"\"\"make submission dataframe for segmentation task\n\n    Args:\n        keys (list[str]): list of keys. key is \"{series_id}_{chunk_id}\"\n        preds (np.ndarray): (num_series * num_chunks, duration, 2)\n\n    Returns:\n        pl.DataFrame: submission dataframe\n    \"\"\"\n    series_ids = np.array(list(map(lambda x: x.split(\"_\")[0], keys)))\n    unique_series_ids = np.unique(series_ids)\n    \n    hm = {}\n    for series_id in unique_series_ids:\n        series_idx = np.where(series_ids == series_id)[0]\n        this_series_preds = preds[series_idx].reshape(-1, 2)\n\n        for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n            tmp_df = []\n            this_event_preds = this_series_preds[:, i]\n            if i == 0:\n                hm[series_id] = {}\n            hm[series_id][event_name] = this_event_preds \n    return hm","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:49.873212Z","iopub.execute_input":"2023-12-05T15:47:49.874156Z","iopub.status.idle":"2023-12-05T15:47:49.88231Z","shell.execute_reply.started":"2023-12-05T15:47:49.874104Z","shell.execute_reply":"2023-12-05T15:47:49.88119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_results_slide_window(pred, gap):\n    scores = list(pred)\n    stack = [0]\n    dp = [-1] * len(scores)\n    dp[0] = 0\n    for i in range(1,len(scores)):\n        if i - stack[-1] < gap:\n            if scores[i] >= scores[stack[-1]]:\n                stack.pop()\n                if i - gap >= 0:\n                    if stack:\n                        if dp[i - gap] != stack[-1]:\n                            while stack and dp[i - gap] - stack[-1] < gap:\n                                stack.pop()\n                            stack.append(dp[i - gap])\n                    else:\n                        stack.append(dp[i - gap])\n                stack.append(i)\n        else:\n            stack.append(i)\n        dp[i] = stack[-1]\n    return stack\n\ndef get_submission(preds,gap,quantile):\n    records = []\n    for series_id in preds[0].keys():\n        for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n            mx_dim = 0\n            for p in preds:\n                tmp_p = p[series_id][event_name]\n                if tmp_p.shape[0] > mx_dim:\n                    mx_dim = tmp_p.shape[0]\n            tmp_df = 0\n            for p in preds:\n                tmp_p = p[series_id][event_name]\n                tmp_df += np.pad(tmp_p, (0, mx_dim - tmp_p.shape[0]), \"constant\")\n\n            steps = get_results_slide_window(tmp_df, gap)\n            scores = tmp_df[steps]\n            tmp_df = []\n            \n            for step, score in zip(steps, scores):\n                tmp_df.append(\n                    {\n                        \"series_id\": series_id,\n                        \"step\": step,\n                        \"event\": event_name,\n                        \"score\": score,\n                    }\n                )\n            tmp_df = pl.DataFrame(tmp_df)\n            tmp_df = tmp_df.filter(pl.col(\"score\") > pl.col(\"score\").quantile(quantile))\n            records.append(tmp_df)\n    if len(records) == 0:\n        records.append(\n            pl.DataFrame({\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            })\n        )\n    sub_df = pl.concat(records)\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:50.953633Z","iopub.execute_input":"2023-12-05T15:47:50.954013Z","iopub.status.idle":"2023-12-05T15:47:50.968906Z","shell.execute_reply.started":"2023-12-05T15:47:50.953981Z","shell.execute_reply":"2023-12-05T15:47:50.968051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/prectime-inference","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:52.431628Z","iopub.execute_input":"2023-12-05T15:47:52.432463Z","iopub.status.idle":"2023-12-05T15:47:52.438679Z","shell.execute_reply.started":"2023-12-05T15:47:52.432428Z","shell.execute_reply":"2023-12-05T15:47:52.437734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"code","source":"!python -m run.prepare_data_prectime dir=kaggle phase=test","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:54.904837Z","iopub.execute_input":"2023-12-05T15:47:54.905298Z","iopub.status.idle":"2023-12-05T15:47:57.941898Z","shell.execute_reply.started":"2023-12-05T15:47:54.905261Z","shell.execute_reply":"2023-12-05T15:47:57.94092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"predictions = []","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:47:57.943668Z","iopub.execute_input":"2023-12-05T15:47:57.943966Z","iopub.status.idle":"2023-12-05T15:47:57.948323Z","shell.execute_reply.started":"2023-12-05T15:47:57.943939Z","shell.execute_reply":"2023-12-05T15:47:57.947333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/inference-prectime-transformers","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:48:00.594895Z","iopub.execute_input":"2023-12-05T15:48:00.595287Z","iopub.status.idle":"2023-12-05T15:48:00.601955Z","shell.execute_reply.started":"2023-12-05T15:48:00.595256Z","shell.execute_reply":"2023-12-05T15:48:00.600925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_list = [\"/kaggle/input/prectime-kfold2/kfold2/fold0\",\n             \"/kaggle/input/prectime-kfold2/kfold2/fold1\",\n             \"/kaggle/input/prectime-kfold2/kfold2/fold2\",\n             \"/kaggle/input/prectime-kfold2/kfold2/fold3\",\n             \"/kaggle/input/prectime-kfold2/kfold2/fold4\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:48:04.773835Z","iopub.execute_input":"2023-12-05T15:48:04.774496Z","iopub.status.idle":"2023-12-05T15:48:04.77892Z","shell.execute_reply.started":"2023-12-05T15:48:04.774465Z","shell.execute_reply":"2023-12-05T15:48:04.778001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor file in file_list:\n    mp = os.path.join(file, \"best_model_weights.pth\")\n    cp = os.path.join(file, \"best_cfg.pkl\")\n    c = f\"python -m run.inference_prectime dir=kaggle phase=test weight_dir={mp} config_dir={cp} post_process.gap=120 post_process.quantile=0.7 overlap_interval=720 chunks=480\"\n    os.system(c)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:48:27.43249Z","iopub.execute_input":"2023-12-05T15:48:27.433206Z","iopub.status.idle":"2023-12-05T15:50:18.490499Z","shell.execute_reply.started":"2023-12-05T15:48:27.433169Z","shell.execute_reply":"2023-12-05T15:50:18.489504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/\nkeys = glob.glob(\"keys*.npy\")\npreds = glob.glob(\"preds*.npy\")\nkeys.sort()\npreds.sort()\nkeys\nfor k, p in tqdm(zip(keys, preds), total = len(keys)):\n    k = np.load(k)\n    p = np.load(p)\n    predictions.append(get_pred_df(k, p[:, :, [1, 2]]))\nfor i, j in zip(keys,preds):\n    c1 = f\"! rm -rf {i}\"\n    c2 = f\"! rm -rf {j}\"\n    os.system(c1)\n    os.system(c2)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:50:20.407658Z","iopub.execute_input":"2023-12-05T15:50:20.408094Z","iopub.status.idle":"2023-12-05T15:50:20.468837Z","shell.execute_reply.started":"2023-12-05T15:50:20.408064Z","shell.execute_reply":"2023-12-05T15:50:20.467909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/inference-prectime-transformers\n\nfile_list = [\"/kaggle/input/sweep-12-3/prectime_kfold_0/single/kaggle-sleep-sweep/h627jpqh\",\n             \"/kaggle/input/sweep-12-3/prectime_kfold_1/single/kaggle-sleep-sweep/9ikd6h7p\",\n             \"/kaggle/input/sweep-12-3/prectime_kfold_2/single/kaggle-sleep-sweep/akzksmjr\",\n             \"/kaggle/input/sweep-12-3/prectime_kfold_3/single/kaggle-sleep-sweep/pny0qi1j\",\n             \"/kaggle/input/sweep-12-3/prectime_kfold_4/single/kaggle-sleep-sweep/4y8td9ur\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-05T08:31:34.429437Z","iopub.execute_input":"2023-12-05T08:31:34.429845Z","iopub.status.idle":"2023-12-05T08:31:34.437321Z","shell.execute_reply.started":"2023-12-05T08:31:34.429793Z","shell.execute_reply":"2023-12-05T08:31:34.436212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor file in file_list:\n    mp = os.path.join(file, \"best_model_weights.pth\")\n    cp = os.path.join(file, \"best_cfg.pkl\")\n    c = f\"python -m run.inference_prectime dir=kaggle phase=test weight_dir={mp} config_dir={cp} post_process.gap=120 post_process.quantile=0.7 overlap_interval=720 chunks=480\"\n    os.system(c)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T08:31:34.438562Z","iopub.execute_input":"2023-12-05T08:31:34.43886Z","iopub.status.idle":"2023-12-05T08:33:05.408207Z","shell.execute_reply.started":"2023-12-05T08:31:34.438812Z","shell.execute_reply":"2023-12-05T08:33:05.4073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/\nkeys = glob.glob(\"keys*.npy\")\npreds = glob.glob(\"preds*.npy\")\nkeys.sort()\npreds.sort()\nkeys\nfor k, p in tqdm(zip(keys, preds), total = len(keys)):\n    k = np.load(k)\n    p = np.load(p)\n    predictions.append(get_pred_df(k, p[:, :, [1, 2]]))\nfor i, j in zip(keys,preds):\n    c1 = f\"! rm -rf {i}\"\n    c2 = f\"! rm -rf {j}\"\n    os.system(c1)\n    os.system(c2)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T08:33:05.409485Z","iopub.execute_input":"2023-12-05T08:33:05.409801Z","iopub.status.idle":"2023-12-05T08:33:05.460279Z","shell.execute_reply.started":"2023-12-05T08:33:05.409774Z","shell.execute_reply":"2023-12-05T08:33:05.459162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/inference-code-sleep/inference-code-sleep","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:51:11.703275Z","iopub.execute_input":"2023-12-05T15:51:11.703642Z","iopub.status.idle":"2023-12-05T15:51:11.71546Z","shell.execute_reply.started":"2023-12-05T15:51:11.703613Z","shell.execute_reply":"2023-12-05T15:51:11.714441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"file_list = [\"/kaggle/input/cv1205/fkyxd3ti\",\n             \"/kaggle/input/cv1205/gml0mpkk\",\n            \"/kaggle/input/cv1205/oqoxaa74\",\n            \"/kaggle/input/cv1205/sokyhi36\",\n            \"/kaggle/input/cv1205/thy6sbe8\"]\n\n# file_list = [\"/kaggle/input/lstm-cv-5-unet-101-1126/kaggle-sleep-sweep/00i4hw45\"]","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:51:12.62818Z","iopub.execute_input":"2023-12-05T15:51:12.628919Z","iopub.status.idle":"2023-12-05T15:51:12.6352Z","shell.execute_reply.started":"2023-12-05T15:51:12.628883Z","shell.execute_reply":"2023-12-05T15:51:12.633926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor file in file_list:\n    mp = os.path.join(file, \"model_weights.pth\")\n    cp = os.path.join(file, \"cfg.pkl\")\n    c = f\"python -m run.inference dir=kaggle phase=test weight_dir={mp} config_dir={cp} post_process.gap=120 post_process.quantile=0.7\"\n    os.system(c)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:51:15.994252Z","iopub.execute_input":"2023-12-05T15:51:15.995104Z","iopub.status.idle":"2023-12-05T15:52:57.72504Z","shell.execute_reply.started":"2023-12-05T15:51:15.99507Z","shell.execute_reply":"2023-12-05T15:52:57.724083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/\nkeys = glob.glob(\"keys*.npy\")\npreds = glob.glob(\"preds*.npy\")\nkeys.sort()\npreds.sort()\nkeys\nfor k, p in tqdm(zip(keys, preds), total = len(keys)):\n    k = np.load(k)\n    p = np.load(p)\n    predictions.append(get_pred_df(k, p[:, :, [1, 2]]))\nfor i, j in zip(keys,preds):\n    c1 = f\"! rm -rf {i}\"\n    c2 = f\"! rm -rf {j}\"\n    os.system(c1)\n    os.system(c2)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:52:57.726595Z","iopub.execute_input":"2023-12-05T15:52:57.72691Z","iopub.status.idle":"2023-12-05T15:52:57.77556Z","shell.execute_reply.started":"2023-12-05T15:52:57.726884Z","shell.execute_reply":"2023-12-05T15:52:57.774695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge","metadata":{}},{"cell_type":"code","source":"sub_df = get_submission(predictions, 120, 0.7)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:54:48.102537Z","iopub.execute_input":"2023-12-05T15:54:48.102958Z","iopub.status.idle":"2023-12-05T15:54:48.152439Z","shell.execute_reply.started":"2023-12-05T15:54:48.102923Z","shell.execute_reply":"2023-12-05T15:54:48.151629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.write_csv(\"./submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T15:54:49.244075Z","iopub.execute_input":"2023-12-05T15:54:49.244461Z","iopub.status.idle":"2023-12-05T15:54:49.2496Z","shell.execute_reply.started":"2023-12-05T15:54:49.244434Z","shell.execute_reply":"2023-12-05T15:54:49.248667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf /kaggle/working/processed_data\n!rm -rf /kaggle/working/output","metadata":{"execution":{"iopub.status.busy":"2023-12-05T08:38:19.760054Z","iopub.execute_input":"2023-12-05T08:38:19.760308Z","iopub.status.idle":"2023-12-05T08:38:21.717482Z","shell.execute_reply.started":"2023-12-05T08:38:19.760286Z","shell.execute_reply":"2023-12-05T08:38:21.716095Z"},"trusted":true},"execution_count":null,"outputs":[]}]}