{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6879418,"sourceType":"datasetVersion","datasetId":3785250},{"sourceId":7061587,"sourceType":"datasetVersion","datasetId":4028375},{"sourceId":144358840,"sourceType":"kernelVersion"},{"sourceId":149254949,"sourceType":"kernelVersion"},{"sourceId":151449137,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"See the following repositories for details  \nhttps://github.com/tubo213/kaggle-child-mind-institute-detect-sleep-states","metadata":{}},{"cell_type":"markdown","source":"# Install additional libraries","metadata":{}},{"cell_type":"code","source":"!pip install hydra-core segmentation_models_pytorch==0.3.3 --no-index --find-links=/kaggle/input/ex-library","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:05.076492Z","iopub.execute_input":"2023-11-27T05:37:05.076841Z","iopub.status.idle":"2023-11-27T05:37:26.799110Z","shell.execute_reply.started":"2023-11-27T05:37:05.076805Z","shell.execute_reply":"2023-11-27T05:37:26.798103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/input/sleep-test/kaggle-child-mind-institute-detect-sleep-states/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:26.800402Z","iopub.execute_input":"2023-11-27T05:37:26.800672Z","iopub.status.idle":"2023-11-27T05:37:26.809362Z","shell.execute_reply.started":"2023-11-27T05:37:26.800649Z","shell.execute_reply":"2023-11-27T05:37:26.808376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /kaggle/input/cmi-code/kaggle-child-mind-institute-detect-sleep-states/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:26.812182Z","iopub.execute_input":"2023-11-27T05:37:26.812467Z","iopub.status.idle":"2023-11-27T05:37:26.820222Z","shell.execute_reply.started":"2023-11-27T05:37:26.812442Z","shell.execute_reply":"2023-11-27T05:37:26.819401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%ls","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:26.821302Z","iopub.execute_input":"2023-11-27T05:37:26.821576Z","iopub.status.idle":"2023-11-27T05:37:27.803477Z","shell.execute_reply.started":"2023-11-27T05:37:26.821542Z","shell.execute_reply":"2023-11-27T05:37:27.802374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Config","metadata":{}},{"cell_type":"code","source":"# Config\nDURATION = 5760\nDOWNSAMPLE_RATE = 2\nPHASE = 'test'\nEXP_NAME = 'exp001'","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:27.804951Z","iopub.execute_input":"2023-11-27T05:37:27.805284Z","iopub.status.idle":"2023-11-27T05:37:27.810243Z","shell.execute_reply.started":"2023-11-27T05:37:27.805254Z","shell.execute_reply":"2023-11-27T05:37:27.809311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess","metadata":{}},{"cell_type":"code","source":"!python -m run.prepare_data dir=kaggle phase=$PHASE","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:27.811410Z","iopub.execute_input":"2023-11-27T05:37:27.811679Z","iopub.status.idle":"2023-11-27T05:37:30.357168Z","shell.execute_reply.started":"2023-11-27T05:37:27.811656Z","shell.execute_reply":"2023-11-27T05:37:30.356070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference\nchange seed","metadata":{}},{"cell_type":"code","source":"!python -m run.inference\\\n    dir=kaggle\\\n    model.encoder_name=resnet34\\\n    model.encoder_weights=null\\\n    num_workers=2\\\n    exp_name=$EXP_NAME\\\n    weight.run_name=single\\\n    batch_size=64\\\n    duration=$DURATION\\\n    downsample_rate=$DOWNSAMPLE_RATE\\\n    post_process.score_th=0.005\\\n    post_process.distance=40\\\n    phase=$PHASE\\\n    seed=42","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:37:30.358587Z","iopub.execute_input":"2023-11-27T05:37:30.358890Z","iopub.status.idle":"2023-11-27T05:38:04.913896Z","shell.execute_reply.started":"2023-11-27T05:37:30.358862Z","shell.execute_reply":"2023-11-27T05:38:04.912863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # いらないファイル、フォルダ削除\n# !rm -rf /kaggle/working/processed_data\n# !rm -rf /kaggle/working/output","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:04.917411Z","iopub.execute_input":"2023-11-27T05:38:04.917751Z","iopub.status.idle":"2023-11-27T05:38:04.922782Z","shell.execute_reply.started":"2023-11-27T05:38:04.917703Z","shell.execute_reply":"2023-11-27T05:38:04.921637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:04.924783Z","iopub.execute_input":"2023-11-27T05:38:04.925588Z","iopub.status.idle":"2023-11-27T05:38:05.889051Z","shell.execute_reply.started":"2023-11-27T05:38:04.925554Z","shell.execute_reply":"2023-11-27T05:38:05.888020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m run.inference\\\n    dir=kaggle\\\n    model.encoder_name=resnet34\\\n    model.encoder_weights=null\\\n    num_workers=2\\\n    exp_name=$EXP_NAME\\\n    weight.run_name=single\\\n    batch_size=64\\\n    duration=$DURATION\\\n    downsample_rate=$DOWNSAMPLE_RATE\\\n    post_process.score_th=0.005\\\n    post_process.distance=40\\\n    phase=$PHASE\\\n    seed=30","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:05.890392Z","iopub.execute_input":"2023-11-27T05:38:05.890677Z","iopub.status.idle":"2023-11-27T05:38:25.456096Z","shell.execute_reply.started":"2023-11-27T05:38:05.890651Z","shell.execute_reply":"2023-11-27T05:38:25.454992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:25.457434Z","iopub.execute_input":"2023-11-27T05:38:25.457756Z","iopub.status.idle":"2023-11-27T05:38:26.444615Z","shell.execute_reply.started":"2023-11-27T05:38:25.457727Z","shell.execute_reply":"2023-11-27T05:38:26.443537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Seed ensemble","metadata":{}},{"cell_type":"code","source":"import numpy as np\nwith open(\"/kaggle/working/keys_30.txt\", 'r') as file:\n    keys_30 = file.read().splitlines()\npreds_30 = np.load(\"/kaggle/working/preds_30.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:26.446043Z","iopub.execute_input":"2023-11-27T05:38:26.446349Z","iopub.status.idle":"2023-11-27T05:38:26.453418Z","shell.execute_reply.started":"2023-11-27T05:38:26.446322Z","shell.execute_reply":"2023-11-27T05:38:26.452436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"/kaggle/working/keys_42.txt\", 'r') as file:\n    keys_42 = file.read().splitlines()\npreds_42 = np.load(\"/kaggle/working/preds_42.npy\")","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:26.454811Z","iopub.execute_input":"2023-11-27T05:38:26.455194Z","iopub.status.idle":"2023-11-27T05:38:26.462992Z","shell.execute_reply.started":"2023-11-27T05:38:26.455167Z","shell.execute_reply":"2023-11-27T05:38:26.462245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = (preds_30 + preds_42)/2\nkeys = keys_30","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:39.869778Z","iopub.execute_input":"2023-11-27T05:39:39.870640Z","iopub.status.idle":"2023-11-27T05:39:39.876523Z","shell.execute_reply.started":"2023-11-27T05:39:39.870602Z","shell.execute_reply":"2023-11-27T05:39:39.875447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# make submission file","metadata":{}},{"cell_type":"code","source":"import polars as pl\nfrom scipy.signal import find_peaks\n\ndef post_process_for_seg(\n    keys: list[str], preds: np.ndarray, score_th: float = 0.01, distance: int = 5000\n) -> pl.DataFrame:\n    \"\"\"make submission dataframe for segmentation task\n\n    Args:\n        keys (list[str]): list of keys. key is \"{series_id}_{chunk_id}\"\n        preds (np.ndarray): (num_series * num_chunks, duration, 2)\n        score_th (float, optional): threshold for score. Defaults to 0.5.\n\n    Returns:\n        pl.DataFrame: submission dataframe\n    \"\"\"\n    series_ids = np.array(list(map(lambda x: x.split(\"_\")[0], keys)))\n    unique_series_ids = np.unique(series_ids)\n\n    records = []\n    for series_id in unique_series_ids:\n        series_idx = np.where(series_ids == series_id)[0]\n        this_series_preds = preds[series_idx].reshape(-1, 2)\n\n        for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n            this_event_preds = this_series_preds[:, i]\n            steps = find_peaks(this_event_preds, height=score_th, distance=distance)[0]\n            scores = this_event_preds[steps]\n\n            for step, score in zip(steps, scores):\n                records.append(\n                    {\n                        \"series_id\": series_id,\n                        \"step\": step,\n                        \"event\": event_name,\n                        \"score\": score,\n                    }\n                )\n\n    if len(records) == 0:  # 一つも予測がない場合はdummyを入れる\n        records.append(\n            {\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            }\n        )\n\n    sub_df = pl.DataFrame(records).sort(by=[\"series_id\", \"step\"])\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:41.319727Z","iopub.execute_input":"2023-11-27T05:39:41.320393Z","iopub.status.idle":"2023-11-27T05:39:41.788290Z","shell.execute_reply.started":"2023-11-27T05:39:41.320358Z","shell.execute_reply":"2023-11-27T05:39:41.787465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_submission(\n    keys: list[str], preds: np.ndarray, downsample_rate, score_th, distance\n) -> pl.DataFrame:\n    sub_df = post_process_for_seg(\n        keys,\n        preds[:, :, [1, 2]],  # type: ignore\n        score_th=score_th,\n        distance=distance,  # type: ignore\n    )\n\n    return sub_df","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:41.789677Z","iopub.execute_input":"2023-11-27T05:39:41.789951Z","iopub.status.idle":"2023-11-27T05:39:41.795266Z","shell.execute_reply.started":"2023-11-27T05:39:41.789927Z","shell.execute_reply":"2023-11-27T05:39:41.794330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = make_submission(\n            keys,\n            preds,\n            downsample_rate=2,\n            score_th=0.005,\n            distance=40,\n        )","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:41.922784Z","iopub.execute_input":"2023-11-27T05:39:41.923095Z","iopub.status.idle":"2023-11-27T05:39:41.931535Z","shell.execute_reply.started":"2023-11-27T05:39:41.923059Z","shell.execute_reply":"2023-11-27T05:39:41.930550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_df = post_process_for_seg(common_ids, preds, score_th=0.005, distance=40)","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:42.170834Z","iopub.execute_input":"2023-11-27T05:39:42.171180Z","iopub.status.idle":"2023-11-27T05:39:42.176265Z","shell.execute_reply.started":"2023-11-27T05:39:42.171151Z","shell.execute_reply":"2023-11-27T05:39:42.175094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# いらないファイル、フォルダ削除\n!rm -rf /kaggle/working/processed_data\n!rm -rf /kaggle/working/output\n!rm -rf /kaggle/working/*.npy\n!rm -rf /kaggle/working/*.txt","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:42.416409Z","iopub.execute_input":"2023-11-27T05:39:42.416725Z","iopub.status.idle":"2023-11-27T05:39:46.354892Z","shell.execute_reply.started":"2023-11-27T05:39:42.416699Z","shell.execute_reply":"2023-11-27T05:39:46.353304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.write_csv(\"/kaggle/working/submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:39:46.357209Z","iopub.execute_input":"2023-11-27T05:39:46.357522Z","iopub.status.idle":"2023-11-27T05:39:46.363601Z","shell.execute_reply.started":"2023-11-27T05:39:46.357493Z","shell.execute_reply":"2023-11-27T05:39:46.362661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !ls /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:26.819688Z","iopub.status.idle":"2023-11-27T05:38:26.820192Z","shell.execute_reply.started":"2023-11-27T05:38:26.819958Z","shell.execute_reply":"2023-11-27T05:38:26.819979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sub_df","metadata":{"execution":{"iopub.status.busy":"2023-11-27T05:38:26.821551Z","iopub.status.idle":"2023-11-27T05:38:26.821879Z","shell.execute_reply.started":"2023-11-27T05:38:26.821716Z","shell.execute_reply":"2023-11-27T05:38:26.821732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}