{"metadata":{"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":6577142,"sourceType":"datasetVersion","datasetId":3798214},{"sourceId":7087798,"sourceType":"datasetVersion","datasetId":4083874},{"sourceId":7103180,"sourceType":"datasetVersion","datasetId":4094757},{"sourceId":7104596,"sourceType":"datasetVersion","datasetId":3954505},{"sourceId":7104600,"sourceType":"datasetVersion","datasetId":3798208},{"sourceId":7116422,"sourceType":"datasetVersion","datasetId":3954338},{"sourceId":7119675,"sourceType":"datasetVersion","datasetId":4106316},{"sourceId":7120867,"sourceType":"datasetVersion","datasetId":4107201},{"sourceId":7122853,"sourceType":"datasetVersion","datasetId":3798198},{"sourceId":144358840,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"papermill":{"default_parameters":{},"duration":91.580556,"end_time":"2023-12-02T07:19:23.56989","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2023-12-02T07:17:51.989334","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"TRAIN_OR_TEST = \"test\"\n\nPENGUIN_PRED_FILE = \"/kaggle/working/penguin_final.parquet\"\nNIKHIL_PRED_FILE = \"/kaggle/working/nikhil_final.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-12-05T16:15:43.596931Z","iopub.execute_input":"2023-12-05T16:15:43.597323Z","iopub.status.idle":"2023-12-05T16:15:43.608274Z","shell.execute_reply.started":"2023-12-05T16:15:43.597292Z","shell.execute_reply":"2023-12-05T16:15:43.607049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n# penguin Part","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/working\n%rm -r /kaggle/working/*\n\n%rm -r /kaggle/output/*\n%cp -r /kaggle/input/zzz-output/output /kaggle/\n%ls /kaggle/output\n\n%cp -r /kaggle/input/zzz-src/src /kaggle/output\n%ls /kaggle/output/src","metadata":{"execution":{"iopub.status.busy":"2023-12-05T16:15:43.609914Z","iopub.execute_input":"2023-12-05T16:15:43.610243Z","iopub.status.idle":"2023-12-05T16:16:15.586699Z","shell.execute_reply.started":"2023-12-05T16:15:43.610217Z","shell.execute_reply":"2023-12-05T16:16:15.585591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## tubo Based Model¶\n","metadata":{}},{"cell_type":"code","source":"!pip install hydra-core segmentation_models_pytorch==0.3.3 --no-index --find-links=/kaggle/input/ex-library\n%cd /kaggle/input/cmi-code/cmi-tubo-base-master/","metadata":{"execution":{"iopub.status.busy":"2023-12-05T16:16:15.588135Z","iopub.execute_input":"2023-12-05T16:16:15.588456Z","iopub.status.idle":"2023-12-05T16:16:37.029052Z","shell.execute_reply.started":"2023-12-05T16:16:15.588429Z","shell.execute_reply":"2023-12-05T16:16:37.027897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m run.prepare_data dir=kaggle phase=test","metadata":{"execution":{"iopub.status.busy":"2023-12-05T16:16:37.031522Z","iopub.execute_input":"2023-12-05T16:16:37.03183Z","iopub.status.idle":"2023-12-05T16:16:39.856907Z","shell.execute_reply.started":"2023-12-05T16:16:37.031803Z","shell.execute_reply":"2023-12-05T16:16:39.855905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport subprocess\nfrom tqdm import tqdm\nimport numpy as np\n\n\n# preprocess\n# Config\nDURATION = 5760\nDOWNSAMPLE_RATE = 2\nPHASE = 'test'\nEXP_NAME = 'exp031'\nTHIS_EXP_NAME = EXP_NAME + f\"_fold{0}\" # 0 is dummy\ncommand = [\n    'python', '-m', 'run.inference',\n    'dir=kaggle',\n    'model.encoder_name=resnet34',\n    'model.encoder_weights=null',\n    'num_workers=2',\n    f'exp_name={THIS_EXP_NAME}',\n    'weight.run_name=single',\n    'batch_size=64',\n    f'duration={DURATION}',\n    f'downsample_rate={DOWNSAMPLE_RATE}',\n    'post_process.score_th=0.005',\n    'post_process.distance=40',\n    f'phase={PHASE}'\n]\nsubprocess.run(command, check=True)\npreds1 = np.load(f\"/kaggle/working/output/inference/{THIS_EXP_NAME}/single/preds.npy\")\nkeys = np.load(f\"/kaggle/working/output/inference/{THIS_EXP_NAME}/single/keys.npy\")\n\n\nDURATION = 5760\nDOWNSAMPLE_RATE = 2\nPHASE = 'test'\nEXP_NAME = 'exp061'\nTHIS_EXP_NAME = EXP_NAME + f\"_fold{0}\" # 0 is dummy\ncommand = [\n    'python', '-m', 'run.inference',\n    'dir=kaggle',\n    'model.encoder_name=resnet34',\n    'model.encoder_weights=null',\n    'num_workers=2',\n    f'exp_name={THIS_EXP_NAME}',\n    'weight.run_name=single',\n    'batch_size=64',\n    f'duration={DURATION}',\n    f'downsample_rate={DOWNSAMPLE_RATE}',\n    'post_process.score_th=0.005',\n    'post_process.distance=40',\n    f'phase={PHASE}'\n]\nsubprocess.run(command, check=True)\npreds2 = np.load(f\"/kaggle/working/output/inference/{THIS_EXP_NAME}/single/preds.npy\")\n\npreds = (preds1 + preds2) / 2\n\n\n# make sub\nimport pandas as pd\nimport gc\nimport sys\nsys.path.append(\"/kaggle/input/cmi-code/cmi-tubo-base-master\")\nfrom src.utils.post_process import post_process_for_seg\n\nsub, oof = post_process_for_seg(\n    keys=keys,\n    preds=preds[:, :, [1, 2]],\n    score_th=0.02,\n    distance=10,\n    penguin_pp=True,\n)\noof = pd.concat(oof)\noof[[\"wakeup_oof\", \"onset_oof\"]] = oof[[\"wakeup_oof\", \"onset_oof\"]].astype(float)\noof.to_parquet(f\"/kaggle/working/tubo_based_oof.parquet\", index=False)\n\ndel oof\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-05T16:16:39.858402Z","iopub.execute_input":"2023-12-05T16:16:39.858703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport yaml\nimport sys\nimport os\nfrom tqdm import tqdm\nimport gc\nfrom pprint import pprint\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings('ignore')\n\nsys.path.append('/kaggle/output/src')\nfrom utils.metric import compute_comptetition_metric\nfrom utils.set_seed import seed_base\n\nPACKAGE_DIR = Path(\"/kaggle/output/src\")\nCFG = yaml.safe_load(open(PACKAGE_DIR / \"config.yaml\", \"r\"))\npprint(CFG)\nseed_base(CFG[\"env\"][\"seed\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## PreProcess","metadata":{}},{"cell_type":"code","source":"# seriesごとのファイルを保存\ntrain = pd.read_parquet(os.path.join(CFG[\"dataset\"][\"competition_dir\"], f\"{TRAIN_OR_TEST}_series.parquet\"))\n\ntrain[\"target\"] = -1 # dummy\n\nif train[\"series_id\"].nunique() == 3:\n    sub = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv\")\n    sub.to_csv(\"/kaggle/working/submission.csv\", index=False)\n    raise\n\nALL_SERIES_IDS  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{TRAIN_OR_TEST}_series.parquet', columns=['series_id'])\nALL_SERIES_IDS = ALL_SERIES_IDS.series_id.unique()\n    \nif TRAIN_OR_TEST == \"train\":\n    cv_split = pd.read_csv(CFG['dataset']['cv_split_path'])\n    train[\"fold\"] = train[\"series_id\"].map(cv_split.set_index(\"series_id\")[\"fold\"])\n    \n    if True: # only fold0\n        train = train[train[\"fold\"]==0].reset_index(drop=True)\n        cv_split = cv_split[cv_split[\"fold\"]==0]\n    series_ids = cv_split[\"series_id\"].unique()\n    train = train[train[\"series_id\"].isin(series_ids)].reset_index(drop=True)\n    \n    if True: # fix num_series\n        num_series = 2\n        series_ids = series_ids[:num_series]\n        train = train[train[\"series_id\"].isin(series_ids)].reset_index(drop=True)\n    ALL_SERIES_IDS = series_ids\n\nfor series_id, df in tqdm(train.groupby(\"series_id\")):\n    path = f\"/kaggle/working/csvs/{series_id}.parquet\"\n    df[['onset_target', 'wakeup_target']] = 1 # dummy\n    os.makedirs(os.path.dirname(path), exist_ok=True)\n    df.to_parquet(path, index=False)\n\ndel train\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime as dt\nfrom datetime import timedelta\nfrom pathlib import Path\n\nfrom matplotlib.dates import date2num\nimport polars as pl\nfrom matplotlib import pyplot as plt\n\n\nDATA_PATH = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")\nTRAIN_SERIES_PATH = DATA_PATH / f\"{TRAIN_OR_TEST}_series.parquet\"\n\nfor path in [\n    DATA_PATH,\n    TRAIN_SERIES_PATH,\n    # SLEEP_DATA_PATH,\n]:\n    assert path.exists()\n    \n    \nimport polars as pl\n\n\ndef transform(df, night_offset=20):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"timestamp\").str.slice(-5, 3).cast(pl.Int8).alias(\"tz_offset\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tz_offset\") == -4).alias(\"is_dst\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"timestamp\")\n                .str.strptime(pl.Datetime, format=\"%Y-%m-%dT%H:%M:%S%z\")\n                .alias(\"timestamp\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"timestamp\").dt.year() - 2000).cast(pl.Int8).alias(\"year\"),\n                pl.col(\"timestamp\").dt.month().cast(pl.Int8).alias(\"month\"),\n                pl.col(\"timestamp\").dt.day().cast(pl.Int8).alias(\"day\"),\n                pl.col(\"timestamp\").dt.hour().cast(pl.Int8).alias(\"hour\"),\n                pl.col(\"timestamp\").dt.minute().cast(pl.Int8).alias(\"minute\"),\n                pl.col(\"timestamp\").dt.second().cast(pl.Int8).alias(\"second\"),\n                pl.col(\"timestamp\").dt.weekday().cast(pl.Int8).alias(\"weekday\"),\n            ]\n        )\n        .with_columns(  # 正午をまたいで日付を調整\n            pl.when(pl.col(\"hour\") < night_offset)\n            .then(pl.col(\"timestamp\"))\n            .otherwise(pl.col(\"timestamp\") + pl.duration(days=1))\n            .dt.date()\n            .alias(\"night_group\"),\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"series_id\")\n                    + pl.lit(\"_\")\n                    + pl.col(\"night_group\").cast(pl.Datetime).dt.strftime(\"%Y%m%d\")\n                ).alias(\"group_id\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"timestamp\").cumcount().over(\"group_id\").alias(\"norm_step\"),\n            ]\n        )\n        .drop([\"night_group\"])\n    )\n\n\ndef transform_series(df):\n    return transform(df).with_columns(\n        [\n            (pl.col(\"enmo\") == 0).alias(\"is_enmo_clipped\"),\n        ]\n    )\n\n\ndef transform_events(df):\n    return (\n        transform(df)\n        .with_columns(\n            [\n                pl.col(\"night\").cast(pl.UInt32).alias(\"night\"),\n            ]\n        )\n        .pivot([\"step\", \"timestamp\", \"tz_offset\"], [\"series_id\", \"group_id\", \"night\"], \"event\")\n    )\n\n\ndef add_feature(\n    df,\n    group_col=\"series_id\",\n    day_group_col=\"group_id\",\n    term1=(5 * 60) // 5,\n    term2=(30 * 60) // 5,\n    term3=(60 * 60) // 5,\n    min_threshold=0.005,\n    max_threshold=0.04,\n    center=True,\n):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"anglez\").diff(1).abs().over(group_col).alias(\"anglez_diff\"),\n                pl.col(\"enmo\").diff(1).abs().over(group_col).alias(\"enmo_diff\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .over(group_col)\n                .alias(\"anglez_diff_median_5min\"),\n                pl.col(\"enmo_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .over(group_col)\n                .alias(\"enmo_diff_median_5min\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff_median_5min\")\n                .quantile(0.1)\n                .clip(min_threshold, max_threshold)\n                .over(day_group_col)\n                .alias(\"critical_threshold\")\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"anglez_diff_median_5min\") < pl.col(\"critical_threshold\") * 15)\n                .over(group_col)\n                .alias(\"is_static\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"is_static\")\n                .cast(pl.Int32)\n                .rolling_sum(term2, center=center)\n                .over(group_col)\n                .alias(\"is_static_sum_30min\"),\n            ]\n        )\n        .with_columns(\n            [(pl.col(\"is_static_sum_30min\") == ((30 * 60) // 5)).over(group_col).alias(\"tmp\")]\n        )\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term2 // 2).over(group_col).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term2 // 2)).over(group_col).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_sleep_block\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_sleep_block\").not_().alias(\"is_gap\")])\n        .with_columns(\n            [\n                pl.col(\"is_gap\")\n                .cast(pl.Int32)\n                .rolling_sum(term3, center=center)\n                .over(group_col)\n                .alias(\"gap_length\")\n            ]\n        )\n        .with_columns([(pl.col(\"gap_length\") == term3).over(group_col).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term3 // 2).over(group_col).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term3 // 2)).over(group_col).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_large_gap\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_large_gap\").not_().alias(\"is_sleep_episode\")])\n        #\n        # extract longest sleep episode\n        #\n        .with_columns(\n            [\n                # extract false->true transition\n                (\n                    (\n                        pl.col(\"is_sleep_episode\")\n                        & pl.col(\"is_sleep_episode\")\n                        .shift_and_fill(pl.lit(False), periods=1)\n                        .not_()\n                    )\n                    .cumsum()\n                    .over(\"group_id\")\n                ).alias(\"sleep_episode_id\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"is_sleep_episode\")\n                .sum()\n                .over([\"group_id\", \"sleep_episode_id\"])\n                .alias(\"sleep_episode_length\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"sleep_episode_length\")\n                .max()\n                .over([\"group_id\"])\n                .alias(\"max_sleep_episode_length\")\n            ]\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"is_sleep_episode\")\n                    & (pl.col(\"sleep_episode_length\") == pl.col(\"max_sleep_episode_length\"))\n                ).alias(\"is_longest_sleep_episode\")\n            ]\n        )\n    )\n\n\n\ntr_series = pl.read_parquet(TRAIN_SERIES_PATH)\ntr_series = transform_series(tr_series)\ntr_series = add_feature(tr_series)\n\nuse_columns = [\n    \"series_id\", \"step\", \n    \"is_longest_sleep_episode\", \"is_sleep_block\", \"is_gap\", \"is_large_gap\", \n    \"is_sleep_episode\", \"is_static\"]\ntr_series = tr_series[use_columns].fill_null(False)\n\n\nfor sid, sdf in tr_series.group_by(\"series_id\"):\n    sdf = sdf.sort(\"step\")\n    path = Path(f\"/kaggle/working/heuristic_features/{sid}.parquet\")\n    path.parent.mkdir(parents=True, exist_ok=True)\n    sdf.write_parquet(path)\n    \ndel tr_series\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!free -h","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n## Patch Model (Transformer/Wavenet/1DCNN + GRU)","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport pandas as pd\nimport yaml\nimport sys\nimport os\nfrom glob import glob\nimport gc\nfrom pathlib import Path\nimport pickle\nfrom tqdm import tqdm\nimport numpy as np\nfrom multiprocessing import Pool\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torch.nn as nn\nimport torch.cuda.amp as amp\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torch.optim.lr_scheduler import ReduceLROnPlateau\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping\nfrom torchmetrics import MetricCollection, MeanSquaredError\nfrom typing import List\n\nsys.path.append('/kaggle/output/src')\nfrom utils.metric import compute_comptetition_metric\nfrom utils.set_seed import seed_base_torch\nfrom utils.feature_contena import Features\nfrom utils.lightning_utils import MyLightningDataModule, MyLightningModule\nfrom utils.torch_template import EnsembleModel\nfrom utils.postprocess import dynamic_range_nms\nfrom torch_models.dataset import ZzzPatchDataset\nfrom torch_models.models import ZzzTransformerGRUModel, ZzzWaveGRUModel, ZzzConv1dGRUModel\nfrom consts import ANGLEZ_MEAN, ANGLEZ_STD, ENMO_MEAN, ENMO_STD\n\nMODEL_NAME = \"patch_transformer_gru\"\n\nPACKAGE_DIR = Path(\"/kaggle/output/src\")\nCFG = yaml.safe_load(open(PACKAGE_DIR / \"config.yaml\", \"r\"))\nprint(CFG[MODEL_NAME][\"execution\"][\"best_exp_id\"])\nBLOCK_SIZE = CFG[MODEL_NAME][\"execution\"][\"block_size\"]\n\nCFG[\"output_dir\"] = f\"/kaggle/output/{CFG[MODEL_NAME]['execution']['best_exp_id']}\"\n\nseed_base_torch(CFG[\"env\"][\"seed\"])\n\nDEVICE = \"cuda\"\n\n\nfiles = glob(\"/kaggle/working/csvs/*.parquet\")\n\nfeatures = Features()\nfeatures.add_num_features([\"anglez\", \"enmo\"])\nfeatures.add_num_features([\"anglez_diff\", \"enmo_diff\"])\nfeatures.add_num_features([\"same_count\"])\nfeatures.add_num_features([\"large_diff_count\"])\nfeatures.add_num_features([\"same_count_shift_plus\", \"same_count_shift_minus\"])\nfeatures.add_num_features([\"is_longest_sleep_episode\", \"is_sleep_block\"])\n# features.add_num_features([\"total_seconds\"])\n\n# transformer + gru\nmodel = ZzzTransformerGRUModel(\n    max_len=BLOCK_SIZE // CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    input_numerical_size=len(features.all_features()) * CFG[MODEL_NAME][\"execution\"][\"patch_size\"],\n    **CFG[MODEL_NAME][\"params\"],\n)\ntrn_models = [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(CFG[\"output_dir\"], f\"logs/best_model_fold{fold}.ckpt\"), \n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(files) > 100 else 1)\n] + [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(f\"/kaggle/output/exp_160\", f\"logs/best_model_fold{fold}.ckpt\"), \n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(files) > 100 else 1)\n]\n\n# wavenet + gru\nMODEL_NAME2 = \"patch_wavenet_gru\"\nCFG[\"output_dir2\"] = f\"/kaggle/output/{CFG[MODEL_NAME2]['execution']['best_exp_id']}\"\nmodel = ZzzWaveGRUModel(\n    input_numerical_size=len(features.all_features()) * CFG[MODEL_NAME2][\"execution\"][\"patch_size\"],\n    **CFG[MODEL_NAME2][\"params\"],\n)\nwave_models = [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(CFG[\"output_dir2\"], f\"logs/best_model_fold{fold}.ckpt\"), \n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(files) > 100 else 1)\n]\n\n# 1dcnn + gru\nMODEL_NAME3 = \"patch_1dcnn_gru\"\nCFG[\"output_dir3\"] = f\"/kaggle/output/{CFG[MODEL_NAME3]['execution']['best_exp_id']}\"\nmodel = ZzzConv1dGRUModel(\n    input_numerical_size=len(features.all_features()) * CFG[MODEL_NAME3][\"execution\"][\"patch_size\"],\n    **CFG[MODEL_NAME3][\"params\"],\n)\ncnn_models = [\n    MyLightningModule.load_from_checkpoint(\n        os.path.join(CFG[\"output_dir3\"], f\"logs/best_model_fold{fold}.ckpt\"), \n        model=model,\n        map_location=torch.device(DEVICE),\n    ).to(DEVICE)\n    for fold in range(5 if len(files) > 100 else 1)\n]\n\nmodels = trn_models + wave_models + cnn_models\nmodel = EnsembleModel(models).to(DEVICE)\nmodel.eval()\n\nsub_dfs = []\nall_oof_dfs = []\nfor file in tqdm(files):\n    # load file\n    df  = pd.read_parquet(file)\n    \n    # same_count\n    DAY_STEPS = 12 * 60 * 24\n    n_days = int(len(df) // DAY_STEPS) + 1\n    df[\"same_count\"] = 0\n    for day in range(-n_days, n_days + 1):\n        if day == 0:\n            continue\n        df[\"_anglez_diff\"] = df[\"anglez\"].diff(DAY_STEPS * day)\n        df[\"_anglez_diff\"] = df[\"_anglez_diff\"].fillna(1)\n        df[\"same_count\"] += (df[\"_anglez_diff\"] == 0).astype(int)\n    df[\"same_count\"] = (df[\"same_count\"].clip(0, 5) - 2.5) / 2.5    \n\n    SHIFT_STEPS = 12 * 60 * 6 # 6h\n    df[\"same_count_shift_plus\"] = df[\"same_count\"].shift(SHIFT_STEPS).fillna(1.0).astype(np.float16)\n    df[\"same_count_shift_minus\"] = df[\"same_count\"].shift(-SHIFT_STEPS).fillna(1.0).astype(np.float16)\n    \n#     # time    \n#     timestamp = pd.to_datetime(df[\"timestamp\"].values[0])\n#     total_seconds = (timestamp - timestamp.replace(hour=0, minute=0, second=0, microsecond=0)).total_seconds()\n#     df[\"total_seconds\"] = (total_seconds + df.index * 5) % (24 * 60 * 60)  # [sec]\n#     df[\"total_seconds\"] /= 24 * 60 * 60  # [day]\n\n    # 特徴量\n    df[\"anglez_diffabs\"] = df[\"anglez\"].diff().abs().fillna(0)\n    df[\"large_diff\"] = (df[\"anglez_diffabs\"] > 5).astype(int)\n    df[\"large_diff_count\"] = df[\"large_diff\"].rolling(10, center=True).mean().fillna(0)\n    df[\"large_diff_count\"] = (df[\"large_diff_count\"] - 0.5) * 2\n\n    # 標準化\n    df[\"anglez\"] = (df[\"anglez\"] - ANGLEZ_MEAN) / ANGLEZ_STD\n    df[\"enmo\"] = (df[\"enmo\"] - ENMO_MEAN) / ENMO_STD\n    df[\"anglez_diff\"] = df[\"anglez\"].diff().fillna(0)\n    df[\"enmo_diff\"] = df[\"enmo\"].diff().fillna(0)\n    \n    # heuristic_features\n    sid = df[\"series_id\"].values[0]\n    path = f\"/kaggle/working/heuristic_features/{sid}.parquet\"\n    hdf = pd.read_parquet(path)\n    df = pd.concat([df, hdf.drop(columns=[\"series_id\", \"step\"])], axis=1)\n    df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] = df[[\"is_longest_sleep_episode\", \"is_sleep_block\"]] * 2 - 1    \n\n    \n    # BLOCK_SIZE ごとに分割\n    dfs = []\n    df = df.sort_values(\"step\").reset_index(drop=True)\n    for start in range(0, len(df), BLOCK_SIZE // 8):\n        end = start + BLOCK_SIZE\n        if end > len(df):\n            # endをlen(df)未満の最大のpsの倍数にする\n            end = len(df) - len(df) % CFG[MODEL_NAME][\"execution\"][\"patch_size\"]\n            start = end - BLOCK_SIZE\n            assert start >= 0\n        assert df.iloc[start][\"step\"] % CFG[MODEL_NAME][\"execution\"][\"patch_size\"] == 0\n        dfs.append(df.iloc[start: end])\n    gc.collect()\n    \n    # 推論\n    # 学習・評価データ\n    train_dataset = ZzzPatchDataset(dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"])\n    valid_dataset = ZzzPatchDataset(dfs, mode=\"test\", features=features, patch_size=CFG[MODEL_NAME][\"execution\"][\"patch_size\"])\n    data_module = MyLightningDataModule(train_dataset, valid_dataset, batch_size=64)\n\n    # 推論\n    preds = []\n    with torch.no_grad():\n        for X in data_module.val_dataloader():\n            pred = torch.sigmoid(model(X.to(\"cuda\"))).detach().cpu().numpy() * 10\n            preds.append(pred)\n\n    oof_dfs = []\n    for pred, df in zip(np.vstack(preds), dfs):\n        df = df.iloc[CFG[MODEL_NAME][\"execution\"][\"patch_size\"] // 2: len(df): CFG[MODEL_NAME][\"execution\"][\"patch_size\"]].reset_index(drop=True)\n        df[[\"wakeup_oof\", \"onset_oof\"]] = pred\n        oof_dfs.append(df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]])\n\n    oof_df = pd.concat(oof_dfs)\n    oof_df = oof_df.groupby([\"series_id\", \"step\"]).mean().reset_index().sort_values([\"series_id\", \"step\"])\n    oof_df = oof_df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]]\n    oof_df[\"step\"] = oof_df[\"step\"].astype(int)\n\n    del preds, oof_dfs\n    gc.collect()\n\n    train = oof_df.reset_index(drop=True)\n    all_oof_dfs.append(train[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]])\n    \n    dfs = []\n    df = train[[\"series_id\", \"step\", \"wakeup_oof\"]].copy()\n    df[\"event\"] = \"wakeup\"\n    df[\"score\"] = df[\"wakeup_oof\"]\n    dfs.append(df[['series_id', 'step', 'event', 'score']])\n\n    df = train[[\"series_id\", \"step\", \"onset_oof\"]].copy()\n    df[\"event\"] = \"onset\"\n    df[\"score\"] = df[\"onset_oof\"]\n    dfs.append(df[['series_id', 'step', 'event', 'score']])\n\n    train = pd.concat(dfs)\n    del dfs, df\n    gc.collect()\n    \n    # Dynamic-Range NMS\n    train = train[train[\"score\"] > 0.005]\n    if len(train) > 0:\n        sub = dynamic_range_nms(train)\n        sub[\"score\"] = sub[\"reduced_score\"]\n        sub_dfs.append(sub)\n    \noof = pd.concat(all_oof_dfs).reset_index(drop=True)\noof.to_parquet(os.path.join(CFG[\"output_dir\"], \"oof.parquet\"), index=False)\n\nsub = pd.concat(sub_dfs)\nsub.to_csv(os.path.join(CFG[\"output_dir\"], \"submission.csv\"), index=False)\ngc.collect()\n\n\n# next_cands\noof[\"oof_max\"] = oof[[\"wakeup_oof\", \"onset_oof\"]].max(axis=1)\noof[\"step\"] = oof[\"step\"].astype(int)\n\nnext_cand_size = 0\ncount = 0\nnext_dict = {}\nfor series_id, train_df in oof.groupby(\"series_id\"):\n    train_df = train_df[train_df[\"oof_max\"] >= 0.01]\n    pred_steps = train_df[\"step\"].values\n\n    if len(train_df) == 0:\n        continue\n    next_cand = np.zeros(pred_steps.max() + 100)\n    for pred_step in pred_steps:\n        next_cand[int(pred_step - 12): int(pred_step + 12)] = 1\n    next_cand_size += np.sum(next_cand)\n    next_dict[series_id] = np.where(next_cand)[0]\nprint(f\"next_cand_size: {next_cand_size}\")\n\nwith open(f\"{CFG['output_dir']}/next_cands.pkl\", \"wb\") as f:\n    pickle.dump(next_dict, f)\n    \ndel sub_dfs, all_oof_dfs, next_dict, oof, sub\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# スコア計算\nif TRAIN_OR_TEST == \"train\":\n    CFG[\"output_dir\"] = f\"/kaggle/output/{CFG['patch_transformer_gru']['execution']['best_exp_id']}\"\n    sub = pd.read_csv(f\"{CFG['output_dir']}/submission.csv\")\n    labels = pd.read_csv(f\"{CFG['dataset']['competition_dir']}/train_events.csv\").dropna()\n    labels = labels[labels[\"series_id\"].isin(sub[\"series_id\"].unique())].reset_index(drop=True)\n    score, ap_table = compute_comptetition_metric(labels, sub)\n    print(f\"score: {score:.4f}\")\n    display(ap_table)\n    \ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!free -h","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GBDT (LightGBM/XGBoost)","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport numpy as np\nimport pandas as pd\nimport yaml\nimport sys\nimport os\nfrom tqdm import tqdm\nimport pickle\nfrom glob import glob\nfrom multiprocessing import Pool\nimport gc\nfrom pathlib import Path\nimport joblib\nimport lightgbm as lgbm\nimport xgboost as xgb\nimport warnings\nwarnings.filterwarnings('ignore')\n\nPACKAGE_DIR = Path(\"/kaggle/output/src\")\nCFG = yaml.safe_load(open(PACKAGE_DIR / \"config.yaml\", \"r\"))\n\nsys.path.append('/kaggle/output/src')\nfrom utils.lgb import inference_lgb\nfrom utils.xgb import inference_xgb\nfrom utils.metric import compute_comptetition_metric\nfrom utils.postprocess import dynamic_range_nms\nfrom utils.set_seed import seed_base\nfrom feature_engineering.fe_xgb import generate_features\n\nMODEL_NAME = \"lgb_model\"\nCFG[\"output_dir\"] = f\"/kaggle/output/{CFG[MODEL_NAME]['execution']['best_exp_id']}\"\n\n# load feature contena\nwith open(os.path.join(CFG[\"output_dir\"], \"features.pkl\"), \"rb\") as f:\n    features = pickle.load(f)\n\n# 特徴生成 -> 推論\nfiles = glob(\"/kaggle/working/csvs/*.parquet\")\n\n# load models\nlgb_wakeup_models = []\nlgb_onset_models = []\nfor i in range(1 if len(files) < 100 else 5):\n    \n    model = joblib.load(os.path.join(CFG[\"output_dir\"], f\"lgb_wakeup_fold_{i}.pkl\"))\n    lgb_wakeup_models.append(model)\n    \n    model = joblib.load(os.path.join(CFG[\"output_dir\"], f\"lgb_onset_fold_{i}.pkl\"))\n    lgb_onset_models.append(model)\n\nMODEL_NAME2 = \"xgb_model\"\nCFG[\"output_dir2\"] = f\"/kaggle/output/{CFG[MODEL_NAME2]['execution']['best_exp_id']}\"\nxgb_wakeup_models = []\nxgb_onset_models = []\nfor i in range(1 if len(files) < 100 else 5):\n    model = xgb.Booster()\n    model.load_model(os.path.join(CFG[\"output_dir2\"], f\"xgb_wakeup_fold_{i}.model\"))\n    xgb_wakeup_models.append(model)\n\n    model = xgb.Booster()\n    model.load_model(os.path.join(CFG[\"output_dir2\"], f\"xgb_onset_fold_{i}.model\"))\n    xgb_onset_models.append(model)\n    \n    \ndfs = []\nfor start in tqdm(range(0, len(files), 4)):\n    end = min(start + 4, len(files))\n\n    # 特徴生成\n    train, _ = generate_features(files[start:end])\n    \n    for event in [\"wakeup\", \"onset\"]:\n        # 推論\n        models = lgb_wakeup_models if event==\"wakeup\" else lgb_onset_models\n        lgb_trn_oof = inference_lgb(models, train[features.all_features()])\n        \n        models = xgb_wakeup_models if event==\"wakeup\" else xgb_onset_models\n        xgb_trn_oof = inference_xgb(models, train[features.all_features()])\n        \n        trn_oof = (lgb_trn_oof + xgb_trn_oof) / 2        \n        train[\"score\"] = trn_oof\n        train[\"event\"] = event\n        dfs.append(train[[\"series_id\", \"step\", \"score\", \"event\"]])\n    del train\n    gc.collect()\ntrain = pd.concat(dfs).reset_index(drop=True)\ntrain = train.sort_values([\"series_id\", \"step\"]).reset_index(drop=True)\ntrain[\"step\"] = train[\"step\"].astype(int)\ntrain.to_parquet(os.path.join(CFG[\"output_dir\"], \"oof.parquet\"), index=False)\n\n# 後処理\ngroups = [group for _, group in train.groupby(\"series_id\")]\nwith Pool(4) as p:  \n    results = list(tqdm(p.imap(dynamic_range_nms, groups), total=len(groups)))\nsub = pd.concat(results)\nsub[\"score\"] = sub[\"reduced_score\"]\nsub.to_csv(os.path.join(CFG[\"output_dir\"], \"submission.csv\"), index=False)\n\ndel train, sub, groups, dfs\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# スコア計算\nif TRAIN_OR_TEST == \"train\":\n    CFG[\"output_dir\"] = f\"/kaggle/output/{CFG['lgb_model']['execution']['best_exp_id']}\"\n    sub = pd.read_csv(f\"{CFG['output_dir']}/submission.csv\")\n    labels = pd.read_csv(f\"{CFG['dataset']['competition_dir']}/train_events.csv\").dropna()\n    labels = labels[labels[\"series_id\"].isin(sub[\"series_id\"].unique())].reset_index(drop=True)\n    score, ap_table = compute_comptetition_metric(labels, sub)\n    print(f\"score: {score:.4f}\")\n    display(ap_table)\n    \ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!free -h","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----\n## Ensemble (only penguin parts)","metadata":{}},{"cell_type":"code","source":"CFG[\"output_dir\"] = f\"/kaggle/output/{CFG['ensemble']['execution']['best_exp_id']}\"\nos.makedirs(CFG[\"output_dir\"], exist_ok=True)\n\noof_lgb = pd.read_parquet(f'/kaggle/output/{CFG[\"lgb_model\"][\"execution\"][\"best_exp_id\"]}/oof.parquet')\noof_gru = pd.read_parquet(f'/kaggle/output/{CFG[\"patch_transformer_gru\"][\"execution\"][\"best_exp_id\"]}/oof.parquet')\noof_tubo_based = pd.read_parquet(f'/kaggle/working/tubo_based_oof.parquet')\n\ndfs = []\nfor seires_id, df_tubo_based in tqdm(oof_tubo_based.groupby(\"series_id\")):\n    df_gru = oof_gru[oof_gru[\"series_id\"]==seires_id]\n    df_lgb = oof_lgb[oof_lgb[\"series_id\"]==seires_id]\n    df = df_tubo_based\n\n    # tubo based\n    df[\"onset_oof_tubo_based\"] = df[\"onset_oof\"] / 1.5\n    df[\"wakeup_oof_tubo_based\"] = df[\"wakeup_oof\"] / 1.5\n\n    # gru\n    df[\"onset_oof_gru\"] = df[\"step\"].map(df_gru.set_index(\"step\")[\"onset_oof\"]).interpolate(method=\"linear\", direction=\"both\", limit=12*10).fillna(0)\n    df[\"wakeup_oof_gru\"] = df[\"step\"].map(df_gru.set_index(\"step\")[\"wakeup_oof\"]).interpolate(method=\"linear\", direction=\"both\", limit=12*10).fillna(0)\n\n    # lgb\n    lgb_wakeup = df_lgb[df_lgb[\"event\"]==\"wakeup\"]\n    df[\"wakeup_oof_lgb\"] = df[\"step\"].map(lgb_wakeup.set_index(\"step\")[\"score\"]).interpolate(method=\"linear\", direction=\"both\", limit=12*10).fillna(0)\n    lgb_onset = df_lgb[df_lgb[\"event\"]==\"onset\"]\n    df[\"onset_oof_lgb\"] = df[\"step\"].map(lgb_onset.set_index(\"step\")[\"score\"]).interpolate(method=\"linear\", direction=\"both\", limit=12*10).fillna(0)\n    \n    df[\"wakeup_oof\"] = (\n          df[\"wakeup_oof_gru\"] * 0.5 \n        + df[\"wakeup_oof_lgb\"] * 0.2 \n        + df[\"wakeup_oof_tubo_based\"] * 0.3\n    )\n    df[\"onset_oof\"] = (\n          df[\"onset_oof_gru\"] * 0.5\n        + df[\"onset_oof_lgb\"] * 0.2\n        + df[\"onset_oof_tubo_based\"] * 0.3\n    )\n\n    df = df[[\"series_id\", \"step\", \"wakeup_oof\", \"onset_oof\"]].reset_index(drop=True)\n    dfs.append(df)\noof_df = pd.concat(dfs).reset_index(drop=True)\n\n\nimport os\nos.makedirs(os.path.dirname(PENGUIN_PRED_FILE), exist_ok=True)\n\noof_df.to_parquet(PENGUIN_PRED_FILE, index=False)\ndel dfs, oof_lgb, oof_gru, oof_tubo_based, oof_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!free -h\n# ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Nikhil Part","metadata":{}},{"cell_type":"markdown","source":"## !pip install -q polars lightgbm tensorflow-addons","metadata":{"execution":{"iopub.execute_input":"2023-10-25T09:05:02.556085Z","iopub.status.busy":"2023-10-25T09:05:02.555795Z","iopub.status.idle":"2023-10-25T09:05:02.56132Z","shell.execute_reply":"2023-10-25T09:05:02.560175Z","shell.execute_reply.started":"2023-10-25T09:05:02.556057Z"},"papermill":{"duration":0.010532,"end_time":"2023-12-02T07:17:55.408097","exception":false,"start_time":"2023-12-02T07:17:55.397565","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nfrom tqdm import tqdm\nimport gc\nimport lightgbm as lgb\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom sklearn.metrics import mean_squared_error\nimport matplotlib.pyplot as plt\nimport gc\nimport joblib\nimport os\nfrom scipy.signal import find_peaks","metadata":{"papermill":{"duration":13.604631,"end_time":"2023-12-02T07:18:09.022982","exception":false,"start_time":"2023-12-02T07:17:55.418351","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class cfg:\n    models_dir=['/kaggle/input/sleep-model-fm-v13',\n                '/kaggle/input/sleep-model-fm-v15',\n                '/kaggle/input/sleep-model-fm-v20',\n                '/kaggle/input/sleep-model-fm-v21'\n               ]\n                \n    samp_freq=1\n    gaussian_overlay_len = 60\n    std_dev_num = 100*(24/samp_freq)","metadata":{"papermill":{"duration":0.01801,"end_time":"2023-12-02T07:18:09.051432","exception":false,"start_time":"2023-12-02T07:18:09.033422","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.009753,"end_time":"2023-12-02T07:18:09.071292","exception":false,"start_time":"2023-12-02T07:18:09.061539","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform(df, night_offset=20):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"timestamp\")\n                .str.strptime(pl.Datetime, format=\"%Y-%m-%dT%H:%M:%S%Z\")\n                .alias(\"timestamp\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"timestamp\").dt.year() - 2000).cast(pl.Int8).alias(\"year\"),\n                pl.col(\"timestamp\").dt.month().cast(pl.Int8).alias(\"month\"),\n                pl.col(\"timestamp\").dt.day().cast(pl.Int8).alias(\"day\"),\n                pl.col(\"timestamp\").dt.hour().cast(pl.Int8).alias(\"hour\"),\n            ]\n        )\n        .with_columns(  # 正午をまたいで日付を調整\n            pl.when(pl.col(\"hour\") < night_offset)\n            .then(pl.col(\"timestamp\"))\n            .otherwise(pl.col(\"timestamp\") + pl.duration(days=1))\n            .dt.date()\n            .alias(\"night_group\"),\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"series_id\")\n                    + pl.lit(\"_\")\n                    + pl.col(\"night_group\").cast(pl.Datetime).dt.strftime(\"%Y%m%d\")\n                ).alias(\"group_id\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"timestamp\").cumcount().over(\"group_id\").alias(\"norm_step\"),\n            ]\n        )\n        .drop([\"night_group\"])\n    )\n\ndef transform_series(df):\n    return transform(df).with_columns(\n        [\n            (pl.col(\"enmo\") == 0).alias(\"is_enmo_clipped\"),\n        ]\n    )","metadata":{"papermill":{"duration":0.023828,"end_time":"2023-12-02T07:18:09.105227","exception":false,"start_time":"2023-12-02T07:18:09.081399","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_feature(\n    df,\n    group_col=\"series_id\",\n    day_group_col=\"group_id\",\n    term1=(5 * 60) // 5,\n    term2=(30 * 60) // 5,\n    term3=(60 * 60) // 5,\n    min_threshold=0.005,\n    max_threshold=0.04,\n    center=True,\n):\n    return (\n        df.with_columns(\n            [\n                pl.col(\"anglez\").diff(1).abs().over(group_col).alias(\"anglez_diff\"),\n                pl.col(\"enmo\").diff(1).abs().over(group_col).alias(\"enmo_diff\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .over(group_col)\n                .alias(\"anglez_diff_median_5min\"),\n                pl.col(\"enmo_diff\")\n                .rolling_median(term1, center=center)  # 5 min window\n                .over(group_col)\n                .alias(\"enmo_diff_median_5min\"),\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"anglez_diff_median_5min\")\n                .quantile(0.1)\n                .clip(min_threshold, max_threshold)\n                .over(day_group_col)\n                .alias(\"critical_threshold\")\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"anglez_diff_median_5min\") < pl.col(\"critical_threshold\") * 15)\n                .over(group_col)\n                .alias(\"is_static\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"is_static\")\n                .cast(pl.Int32)\n                .rolling_sum(term2, center=center)\n                .over(group_col)\n                .alias(\"is_static_sum_30min\"),\n            ]\n        )\n        .with_columns(\n            [(pl.col(\"is_static_sum_30min\") == ((30 * 60) // 5)).over(group_col).alias(\"tmp\")]\n        )\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term2 // 2).over(group_col).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term2 // 2)).over(group_col).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_sleep_block\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_sleep_block\").not_().alias(\"is_gap\")])\n        .with_columns(\n            [\n                pl.col(\"is_gap\")\n                .cast(pl.Int32)\n                .rolling_sum(term3, center=center)\n                .over(group_col)\n                .alias(\"gap_length\")\n            ]\n        )\n        .with_columns([(pl.col(\"gap_length\") == term3).over(group_col).alias(\"tmp\")])\n        .with_columns(\n            [\n                pl.col(\"tmp\").shift(term3 // 2).over(group_col).alias(\"tmp_left\"),\n                pl.col(\"tmp\").shift(-(term3 // 2)).over(group_col).alias(\"tmp_right\"),\n            ]\n        )\n        .with_columns(\n            [\n                (pl.col(\"tmp_left\") | pl.col(\"tmp_right\")).alias(\"is_large_gap\"),\n            ]\n        )\n        .drop([\"tmp\", \"tmp_left\", \"tmp_right\"])\n        .with_columns([pl.col(\"is_large_gap\").not_().alias(\"is_sleep_episode\")])\n        #\n        # extract longest sleep episode\n        #\n        .with_columns(\n            [\n                # extract false->true transition\n                (\n                    (\n                        pl.col(\"is_sleep_episode\")\n                        & pl.col(\"is_sleep_episode\")\n                        .shift_and_fill(pl.lit(False), periods=1)\n                        .not_()\n                    )\n                    .cumsum()\n                    .over(\"group_id\")\n                ).alias(\"sleep_episode_id\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"is_sleep_episode\")\n                .sum()\n                .over([\"group_id\", \"sleep_episode_id\"])\n                .alias(\"sleep_episode_length\")\n            ]\n        )\n        .with_columns(\n            [\n                pl.col(\"sleep_episode_length\")\n                .max()\n                .over([\"group_id\"])\n                .alias(\"max_sleep_episode_length\")\n            ]\n        )\n        .with_columns(\n            [\n                (\n                    pl.col(\"is_sleep_episode\")\n                    & (pl.col(\"sleep_episode_length\") == pl.col(\"max_sleep_episode_length\"))\n                ).alias(\"is_longest_sleep_episode\")\n            ]\n        )\n    )","metadata":{"papermill":{"duration":0.034519,"end_time":"2023-12-02T07:18:09.149985","exception":false,"start_time":"2023-12-02T07:18:09.115466","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ntest_series = pl.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{TRAIN_OR_TEST}_series.parquet')    \ntest_series = test_series.sort(by=['series_id', 'step'])\ntest_series = transform_series(test_series)\ntest_series = add_feature(test_series)\ntest_series = test_series[['series_id', 'step', 'timestamp', 'anglez', 'enmo', 'is_longest_sleep_episode', 'is_sleep_episode', 'is_large_gap', 'is_gap', 'is_sleep_block', 'is_static']]\ntest_series = test_series.with_columns([pl.col(c).fill_null(False) for c in test_series.columns if c != 'series_id'])\nis_added_cols = ['is_longest_sleep_episode', 'is_sleep_episode', 'is_large_gap', 'is_gap', 'is_sleep_block', 'is_static']\n\ntest_series = test_series.with_columns([(pl.col(c)*1).alias(c) for c in is_added_cols])\n\ntest_series = test_series.with_columns(pl.col('step').cast(pl.Int64))\n\nss = pl.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/sample_submission.csv')\n# model_dct = joblib.load(os.path.join(cfg.model_dir, 'model_dct.pkl'))\n# model = tf.keras.models.load_model(os.path.join(cfg.model_dir, 'first_model.h5'))\n# print(model_dct)","metadata":{"papermill":{"duration":0.207549,"end_time":"2023-12-02T07:18:09.36794","exception":false,"start_time":"2023-12-02T07:18:09.160391","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_dct.keys()","metadata":{"papermill":{"duration":0.017664,"end_time":"2023-12-02T07:18:09.396063","exception":false,"start_time":"2023-12-02T07:18:09.378399","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"partitions = test_series.partition_by(by='series_id', maintain_order=True)","metadata":{"papermill":{"duration":0.020241,"end_time":"2023-12-02T07:18:09.426593","exception":false,"start_time":"2023-12-02T07:18:09.406352","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rolling_nunique(arr, window=5):\n    n = len(arr)\n    nunique = np.zeros_like(arr)\n    if window > n:\n        return nunique\n    \n    # Initialize the dictionary with the first window\n    window_counts = {}\n    for i in arr[:window]:\n        window_counts[i] = window_counts.get(i, 0) + 1\n\n    # Set the unique count for the first window\n    nunique[window-1] = len(window_counts)\n\n    for i in range(window, n):\n        # Element leaving the window\n        leaving = arr[i - window]\n        window_counts[leaving] -= 1\n        if window_counts[leaving] == 0:\n            del window_counts[leaving]\n\n        # Element entering the window\n        entering = arr[i]\n        window_counts[entering] = window_counts.get(entering, 0) + 1\n\n        # Update the unique count\n        nunique[i] = len(window_counts)\n\n    return nunique","metadata":{"papermill":{"duration":0.022206,"end_time":"2023-12-02T07:18:09.459183","exception":false,"start_time":"2023-12-02T07:18:09.436977","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df, add_target=False):\n    \n    df = df.sort(by='step')\n    \n    df = df.with_columns([pl.col('timestamp').dt.hour().alias('hour'),\n                                         (pl.col('timestamp').dt.hour() * 60 + pl.col('timestamp').dt.minute()).alias('hour_minute'),\n                                          (pl.col('timestamp').dt.hour()*60*60 + pl.col('timestamp').dt.minute()*60 + pl.col('timestamp').dt.second()).alias('hms'),\n                                          pl.col('timestamp').dt.weekday().alias('dayofweek'),\n                                          pl.col('enmo').diff().fill_null(0).alias('enmo_diff'),\n                                          pl.col('anglez').diff().abs().alias('anglez_diff').fill_null(0),\n                                          (pl.col('step') % (24*60*12)).alias('step_mod')]).with_columns([\\\n    ((pl.col('anglez_diff') > 5)*1.0).alias('large_anglez_diff'),\n     pl.col('enmo').mean().over(['hour']).alias('enmo_mean_hour'),\n    pl.col('enmo').std().over(['hour']).fill_null(0).alias('enmo_std_hour'),\n    pl.col('enmo').std().over(['hms']).fill_null(0).alias('anglez_std_stepmod'),\n    ((pl.col('enmo') == pl.col('enmo').median())*1).over(['hms']).alias('enmo_eq_median_stepmod'),\n    ((pl.col('enmo') == pl.col('enmo').mean())*1).over(['hms']).alias('enmo_eq_stepmod')\n    ]).with_columns([\n        pl.col('enmo_eq_stepmod').sum().over(['hms']).alias('enmo_eq_cnt_stepmod'),\n        pl.col('enmo_eq_stepmod').mean().over(['hms']).alias('enmo_eq_pct_stepmod')\n    ]).with_columns([\n        pl.col('enmo_eq_pct_stepmod').diff().fill_null(0).alias('enmo_eq_pct_stepmod_diff'),\n    ])\n    \n    windows = [12, 60]\n    \n    df = df.with_columns([pl.Series(rolling_nunique(df['anglez'].to_numpy().astype('int'), window=window)).alias(f'rolling_nunique_anglez_win_{window}')\\\n                            for window in windows])\n    \n    aggs_col_wise = {\n        'enmo': ['mean', 'std'],\n        'anglez_diff': ['max'],\n        'large_anglez_diff': ['max', 'mean']\n    }\n\n    df = df.with_columns([getattr(pl.col(col), f'rolling_{agg}')(window_size=win_size, min_periods=1, center=True)\\\n        .alias(f'{col}_rolling_{agg}_win_{win_size}')\\\n        for col in aggs_col_wise.keys() for agg in aggs_col_wise[col] for win_size in windows])\n    \n    ### Rolling range by calculating max - min\n    \n    df = df.with_columns([(pl.col('anglez').rolling_max(window_size=win_size, min_periods=1, center=True) -\\\n        pl.col('anglez').rolling_min(window_size=win_size, min_periods=1, center=True)).fill_null(0).alias(f'anglez_rolling_range_win_{win_size}')\\\n            for win_size in windows])\n\n    if add_target == True:\n        df = df.join(train_events[['series_id', 'event', 'step']], on=['series_id', 'step'], how='left')\n        df = df.with_columns(pl.col('event').map_dict({'onset': 1, 'wakeup': 0}).fill_null(0).alias('is_onset'))\n        df = df.with_columns(pl.col('event').map_dict({'onset': 0, 'wakeup': 1}).fill_null(0).alias('is_wakeup'))\n\n#     else:\n#         df = df.with_columns([((pl.col(c) - model_dct[c]['mean'])/model_dct[c]['std']).alias(c) for c in model_dct.keys()])\n\n        \n    return df","metadata":{"papermill":{"duration":0.030615,"end_time":"2023-12-02T07:18:09.501222","exception":false,"start_time":"2023-12-02T07:18:09.470607","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del partitions\n_ = gc.collect()","metadata":{"papermill":{"duration":0.261203,"end_time":"2023-12-02T07:18:09.772823","exception":false,"start_time":"2023-12-02T07:18:09.51162","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def resample_arr_1d(arr, samp_freq=12, samp_method='mean'):\n    \n    if len(arr) % samp_freq != 0:\n        n_rem = samp_freq - len(arr) % samp_freq\n        arr = np.concatenate([arr, np.zeros(n_rem)])\n    \n    arr_samp = arr.reshape(-1, samp_freq)\n    \n    if samp_method == 'last':\n        arr_samp = arr_samp[:, -1]\n    elif samp_method in ['argmax', 'argmin']:\n        arr_samp =getattr(arr_samp, samp_method)(axis=1)/cfg.samp_freq\n    \n    elif samp_method == 'first':\n        arr_samp = arr_samp[:, 0]\n        \n    elif samp_method == 'aux':\n        argmax_idx = np.argmax(arr_samp, axis=1)\n        return np.where(argmax_idx > samp_freq//2, 0, 1)\n        \n    else:\n        arr_samp = getattr(arr_samp, samp_method)(axis=1)\n\n    return arr_samp\n\n","metadata":{"papermill":{"duration":0.020593,"end_time":"2023-12-02T07:18:09.803903","exception":false,"start_time":"2023-12-02T07:18:09.78331","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def split_and_pad(X, y=None, split_length=1440, stride=1440):\n    \"\"\"\n    Splits and pads the arrays X and y using a sliding window.\n\n    Parameters:\n    - X (np.array): Array with shape (timesteps, features)\n    - y (np.array): Array with shape (timesteps, 2)\n    - split_length (int): Desired timestep length for the resulting arrays.\n    - stride (int): Step size for sliding window.\n\n    Returns:\n    - List of arrays for X and y, each with shape (split_length, features)\n    \"\"\"\n    \n    if y is not None and len(X) != len(y):\n        raise ValueError(\"X and y should have the same number of timesteps.\")\n    \n    timesteps, features = X.shape\n    \n    # Create empty lists to store split segments\n    X_splits = []\n    y_splits = []\n    starts = []\n    \n    # Use sliding window to extract segments\n    \n    for start in range(0, timesteps, stride):\n        end = start + split_length\n        if end <= timesteps:\n            starts.append(start)\n            X_splits.append(X[start:end].copy())\n            if y is not None:\n                y_splits.append(y[start:end].copy())\n        else:\n            # If the segment is shorter than split_length, pad it\n            starts.append(start)\n            padding_length = end - timesteps\n            X_segment_padded = np.pad(X[start:].copy(), ((0, padding_length), (0, 0)), mode='constant', constant_values=-9)\n            X_splits.append(X_segment_padded)\n            \n            if y is not None:\n                y_segment_padded = np.pad(y[start:].copy(), ((0, padding_length), (0, 0)), mode='constant', constant_values=0)\n                y_splits.append(y_segment_padded)\n                \n            break\n            \n            \n    if y is not None:\n        return X_splits, y_splits, starts\n    \n    return X_splits, starts","metadata":{"papermill":{"duration":0.023853,"end_time":"2023-12-02T07:18:09.837932","exception":false,"start_time":"2023-12-02T07:18:09.814079","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDataset:\n    \n    def __init__(self, X_s, y_s=None, series_ids=None, samp_freq=None, remove_no_dets=True, is_train=True, split_factor=1):\n        \n        self.split_len=(24*60*12)//cfg.samp_freq\n        \n        if is_train:\n            self.split_strides = self.split_len\n        else:\n            self.split_strides = self.split_len       \n        self.remove_no_dets=remove_no_dets\n        self.create_dataset(X_s, y_s, series_ids)\n        \n        \n    def create_dataset(self, X_s, y_s=None, series_ids=None):\n        X_s_splits, y_s_splits, series_splits, starts_splits = [], [], [], []\n\n        for i in range(len(X_s)):\n            x_splits, starts = split_and_pad(X_s[i].copy(), split_length = self.split_len, stride=self.split_strides)\n            X_s_splits.extend(x_splits)\n            starts_splits.extend(starts)\n\n            if y_s is not None:\n                _, y_splits, _ = split_and_pad(X_s[i].copy(), y_s[i].copy(), split_length=self.split_len, stride=self.split_strides)\n                y_s_splits.extend(y_splits)\n\n            if series_ids is not None:\n                series_splits.extend([series_ids[i] for _ in range(len(x_splits))])\n            \n        self.X = np.array(X_s_splits)\n        self.starts_splits = np.array(starts_splits)\n        \n        if y_s is not None:\n            self.y = np.array(y_s_splits)\n\n            if self.remove_no_dets:\n                fltr = (self.y[:, :, 1].sum(axis=1) + self.y[:, :, 0].sum(axis=1)) != 0\n                self.X = self.X[fltr]\n                self.y = self.y[fltr]\n                if series_ids is not None:\n                    self.series_ids = np.array(series_splits)[fltr]\n                else:\n                    self.series_ids = None\n            \n            self.y = np.array([normalize_y(yts) for yts in self.y])\n\n        else:\n            self.y = None\n\n        if series_ids is not None:\n            if not hasattr(self, 'series_ids'):\n                self.series_ids = np.array(series_splits)","metadata":{"papermill":{"duration":0.026125,"end_time":"2023-12-02T07:18:09.87418","exception":false,"start_time":"2023-12-02T07:18:09.848055","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_test_series = test_series['series_id'].unique(maintain_order=True)","metadata":{"papermill":{"duration":0.020765,"end_time":"2023-12-02T07:18:09.905488","exception":false,"start_time":"2023-12-02T07:18:09.884723","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npartitions = test_series.partition_by(by='series_id', maintain_order=True, as_dict=True)\ndel test_series\n\n_ = gc.collect()","metadata":{"papermill":{"duration":0.251344,"end_time":"2023-12-02T07:18:10.167176","exception":false,"start_time":"2023-12-02T07:18:09.915832","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.010043,"end_time":"2023-12-02T07:18:10.188045","exception":false,"start_time":"2023-12-02T07:18:10.178002","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def partition_series_data(df_ser_sub, ft_cols, cfg, target_cols=None, norm_agg_dict=None):\n    df_ser_sub = df_ser_sub.sort(by=['step'])\n    \n    ser_id = df_ser_sub['series_id'][0]\n    \n    # If target_cols is not provided or is empty, skip y_s related computation\n    compute_y_s = target_cols is not None and len(target_cols) > 0\n\n    X_s, series_ids = [], []\n    if compute_y_s:\n        y_s = []\n        \n    X_sub = df_ser_sub[ft_cols].to_numpy()\n    X_sub_samp = np.concatenate([np.array([resample_arr_1d(X_sub[:, i], cfg.samp_freq, agg) for agg in norm_agg_dict[ft_cols[i]]]).T for i in range(X_sub.shape[1])], axis=1)\n\n    X_s.append(X_sub_samp)\n    series_ids.append(ser_id)\n\n    if compute_y_s:\n        y_sub = get_y_gaussian(df_ser_sub)\n        y_sub_samp = np.array([resample_arr_1d(y_sub[:, i], cfg.samp_freq, 'max') for i in range(len(target_cols))]).T\n        y_s.append(y_sub_samp)\n            \n    \n    if compute_y_s:\n        return X_s, y_s, series_ids\n    else:\n        return X_s, series_ids","metadata":{"papermill":{"duration":0.022316,"end_time":"2023-12-02T07:18:10.220652","exception":false,"start_time":"2023-12-02T07:18:10.198336","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"norm_cols = ['enmo',\n 'hour',\n 'anglez',\n 'dayofweek',\n 'enmo_diff',\n 'anglez_diff',\n 'anglez_std_stepmod',\n 'enmo_eq_stepmod',\n 'enmo_eq_pct_stepmod',\n 'enmo_eq_pct_stepmod_diff',\n 'large_anglez_diff',\n 'enmo_eq_median_stepmod',\n 'is_longest_sleep_episode',\n 'is_sleep_episode',\n 'is_large_gap',\n 'is_gap',\n 'is_sleep_block',\n 'is_static',\n 'rolling_nunique_anglez_win_12',\n 'rolling_nunique_anglez_win_60',\n 'enmo_rolling_mean_win_12',\n 'enmo_rolling_mean_win_60',\n 'enmo_rolling_std_win_12',\n 'enmo_rolling_std_win_60',\n 'anglez_diff_rolling_max_win_12',\n 'anglez_diff_rolling_max_win_60',\n 'large_anglez_diff_rolling_max_win_12',\n 'large_anglez_diff_rolling_max_win_60',\n 'large_anglez_diff_rolling_mean_win_12',\n 'large_anglez_diff_rolling_mean_win_60',\n 'anglez_rolling_range_win_12',\n 'anglez_rolling_range_win_60']\n\n\nnorm_agg_dict = {c: ['mean'] for c in norm_cols}","metadata":{"papermill":{"duration":0.018597,"end_time":"2023-12-02T07:18:10.249463","exception":false,"start_time":"2023-12-02T07:18:10.230866","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# assert False","metadata":{"papermill":{"duration":0.017335,"end_time":"2023-12-02T07:18:10.27697","exception":false,"start_time":"2023-12-02T07:18:10.259635","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_model_folds_list(model_dir):\n    \n    paths = [path for path in os.listdir(model_dir) if path.startswith('tf_model_fold_')]\n    folds = [int(path.split('_')[-1][:-3]) for path in paths]\n    folds = sorted(folds)\n    \n    return folds","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\n\nfor output_dir in cfg.models_dir:\n    models[output_dir] = {}\n    models[output_dir]['model_dct'] = joblib.load(os.path.join(output_dir, 'model_dct.pkl'))\n    models[output_dir]['models'] = {}\n    \n    folds = get_model_folds_list(output_dir)\n    print(folds)\n    for fold in folds:\n        model_path = os.path.join(output_dir, f'tf_model_fold_{fold}.h5')\n        model = tf.keras.models.load_model(model_path)\n        models[output_dir]['models'][fold] = model\n        \n_ = gc.collect()","metadata":{"papermill":{"duration":54.801762,"end_time":"2023-12-02T07:19:05.088885","exception":false,"start_time":"2023-12-02T07:18:10.287123","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_series_ids = []\nres_steps = []\nres_preds = []\n\nfor key in tqdm(partitions.keys()):\n    test_ser_sub = partitions[key]\n    test_ser_sub = feat_eng(test_ser_sub)\n    X_s, series_ids  = partition_series_data(test_ser_sub, norm_cols, cfg, norm_agg_dict=norm_agg_dict)\n    \n    test_ds = SleepDataset(X_s, series_ids=series_ids, samp_freq=cfg.samp_freq, remove_no_dets=False, is_train=False, split_factor=1)\n    \n    avg_val_preds_lst = []\n    for output_dir in models.keys():\n        model_dct = models[output_dir]['model_dct']\n        for fold_num, model in models[output_dir]['models'].items():\n            X = test_ds.X.copy()\n            norm_params =  model_dct[fold_num]\n            X = (X - norm_params['mean'])/(norm_params['std'] + 1e-6)\n            avg_val_preds_lst.append(model(X))\n\n    preds = np.mean(avg_val_preds_lst, axis=0)\n    \n    \n    del avg_val_preds_lst\n    _ = gc.collect()\n\n    preds_t = preds.reshape((preds.shape[0]*preds.shape[1]), 2)    \n    steps_t = np.concatenate([test_ds.starts_splits[idx] + np.arange(len(preds[idx])) for idx in range(preds.shape[0])])\n \n    res_preds.append(preds_t)\n    res_steps.append(steps_t)\n    \n    res_series_ids.append(key)","metadata":{"papermill":{"duration":13.67516,"end_time":"2023-12-02T07:19:18.774889","exception":false,"start_time":"2023-12-02T07:19:05.099729","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# res_preds","metadata":{"papermill":{"duration":0.018533,"end_time":"2023-12-02T07:19:18.857741","exception":false,"start_time":"2023-12-02T07:19:18.839208","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del partitions, models, test_ds, model_dct\n_ = gc.collect()","metadata":{"papermill":{"duration":0.401232,"end_time":"2023-12-02T07:19:19.270241","exception":false,"start_time":"2023-12-02T07:19:18.869009","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds[:, :, 0].max(), preds[:, :, 1].max()","metadata":{"papermill":{"duration":0.02174,"end_time":"2023-12-02T07:19:19.303936","exception":false,"start_time":"2023-12-02T07:19:19.282196","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds.shape","metadata":{"papermill":{"duration":0.019857,"end_time":"2023-12-02T07:19:19.334951","exception":false,"start_time":"2023-12-02T07:19:19.315094","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(preds[0][:, 1])","metadata":{"papermill":{"duration":0.326787,"end_time":"2023-12-02T07:19:19.672911","exception":false,"start_time":"2023-12-02T07:19:19.346124","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_pred_df(val_preds, val_series_ids, val_steps):\n    dfs = []\n    \n    for i in np.arange(len(val_preds)):\n        df = pd.DataFrame({\n            \"series_id\": val_series_ids[i],\n            \"step\": val_steps[i],\n            \"onset_oof\": val_preds[i][:, 0],\n            \"wakeup_oof\": val_preds[i][:, 1],\n        })\n        dfs.append(df)\n    oof = pd.concat(dfs).reset_index(drop=True)\n    return oof\n\noof = get_pred_df(res_preds, res_series_ids, res_steps)\n\nimport os\nos.makedirs(os.path.dirname(NIKHIL_PRED_FILE), exist_ok=True)\n\noof.to_parquet(NIKHIL_PRED_FILE, index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del res_preds, res_series_ids, res_steps\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--- \n# Submission","metadata":{}},{"cell_type":"code","source":"pdf = pl.read_parquet(PENGUIN_PRED_FILE)\ndf = pl.read_parquet(NIKHIL_PRED_FILE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pdf = pdf.with_columns(pl.col(c).cast(df[c].dtype) for c in pdf.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.rename({'onset_oof': 'onset_nikhil', 'wakeup_oof': 'wakeup_nikhil'})\npdf = pdf.rename({'onset_oof': 'onset_penguin', 'wakeup_oof': 'wakeup_penguin'})\nprint(df.columns, pdf.columns)","metadata":{"papermill":{"duration":0.012433,"end_time":"2023-12-02T07:19:20.121075","exception":false,"start_time":"2023-12-02T07:19:20.108642","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.join(pdf, on=['series_id', 'step'], how='left')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del pdf\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.with_columns([pl.col('wakeup_penguin').interpolate().over('series_id'),\n                        pl.col('onset_penguin').interpolate().over('series_id')])\ndf = df.with_columns([pl.col('wakeup_penguin').fill_null(strategy='forward').over('series_id'),\n                        pl.col('onset_penguin').fill_null(strategy='forward').over('series_id')])\ndf = df.with_columns([pl.col('wakeup_penguin').fill_null(strategy='backward').over('series_id'),\n                        pl.col('onset_penguin').fill_null(strategy='backward').over('series_id')])\ndf = df.with_columns([pl.col('wakeup_penguin').fill_null(0),\n                        pl.col('onset_penguin').fill_null(0)])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def scale_array(arr, new_min, new_max):\n    \"\"\"\n    Scale a numpy array to a new given range [new_min, new_max].\n\n    :param arr: numpy array to be scaled\n    :param new_min: new minimum value of the range\n    :param new_max: new maximum value of the range\n    :return: scaled numpy array\n    \"\"\"\n    min_val = np.min(arr)\n    max_val = np.max(arr)\n    \n    # Scale the array\n    scaled_arr = ((arr - min_val) / (max_val - min_val)) * (new_max - new_min) + new_min\n    \n    return scaled_arr\n\n\ndef scale(arr, new_min, new_max, power=1.25):\n    arr = scale_array(arr, new_min, new_max)\n    arr = arr**power\n    arr = scale_array(arr, new_min, new_max)\n    return arr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.with_columns([pl.Series(scale(df[c].to_numpy(), 0, 15, 0.7)).alias(c) for c in ['wakeup_penguin', 'onset_penguin']])\ndf = df.with_columns([pl.Series(scale(df[c].to_numpy(), 0, 15, 1)).alias(c) for c in ['wakeup_nikhil', 'onset_nikhil']])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NIKHIL_WT = 0.5\nPENGUIN_WT = (1 - NIKHIL_WT)\n\ndf = df.with_columns([(pl.col('wakeup_penguin') * PENGUIN_WT + pl.col('wakeup_nikhil') * NIKHIL_WT).alias('wakeup_blend'),\n                        (pl.col('onset_penguin') * PENGUIN_WT + pl.col('onset_nikhil') * NIKHIL_WT).alias('onset_blend')])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res_series_ids = []\nres_steps = []\nres_preds = []\n\npartitions = df.partition_by(by='series_id', maintain_order=True)\n\ndel df\n_ = gc.collect()\n\n\nfor df_sub in tqdm(partitions):\n    series_id = df_sub['series_id'][0]\n    res_series_ids.append(series_id)\n    res_steps.append(df_sub['step'].to_numpy())\n    res_preds.append(df_sub[['onset_blend', 'wakeup_blend']].to_numpy())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del partitions\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def wbf_nikhil(preds_orig, max_thresh=0.1, max_count=1000, hyperparams=None):\n    k_dist = hyperparams['k_dist']\n    log_base = hyperparams['log_base']\n    log_scale = hyperparams['log_scale']\n    curr_max_power = hyperparams['curr_max_power']\n    weight_coeff = hyperparams['weight_coeff']\n    convolution_kernel = hyperparams['convolution_kernel']\n    section_weight_method = hyperparams['section_weight_method']\n    preds_reduction_power = hyperparams['preds_reduction_power']\n    overlap_coeff = hyperparams['overlap_coeff']\n    min_distance = hyperparams['min_distance']\n    \n    preds = preds_orig.copy()\n    preds = np.convolve(preds, convolution_kernel, mode='same')\n\n    count = 0\n    indices = []\n    scores = []\n\n    while count < max_count:\n        curr_max_idx = np.argmax(preds)\n        curr_max = preds[curr_max_idx]\n\n        if curr_max < max_thresh:\n            break\n\n        k = int(k_dist - max(min_distance, (curr_max**curr_max_power)))\n\n        start_idx = max(curr_max_idx - k, 0)\n        end_idx = min(curr_max_idx + k + 1, len(preds))\n\n        section = preds[start_idx:end_idx]\n\n        # Different weight calculation methods\n        distances = np.abs(np.arange(len(section)) - k)\n        if section_weight_method == 'logarithmic':\n            weights = 1 / (log_base ** (distances / (k * log_scale)))\n        elif section_weight_method == 'linear':\n            weights = 1 - (distances / k) * weight_coeff\n        # Add more methods as needed\n\n        weighted_avg = np.sum(section * weights) / np.sum(weights)\n\n        scores.append(weighted_avg)\n        indices.append(curr_max_idx)\n\n        preds[start_idx:end_idx] *= ((1 - weights * overlap_coeff))**preds_reduction_power\n\n        count += 1\n\n    return indices, scores\n\ndef get_actual_preds(val_preds, val_series_ids, val_steps, type_, hyperparams):\n    times = []\n    series_ids = []\n    scores = []\n\n    for i in np.arange(len(val_preds)):\n        \n        vp_i = val_preds[i]\n        ser_id = val_series_ids[i]\n        \n        col_index = 0 if type_ == \"onset\" else 1\n\n        preds = vp_i[:, col_index] \n        peaks, peak_scores = wbf_nikhil(preds, hyperparams=hyperparams)\n    # ...\n\n        times.extend(val_steps[i][peaks])\n        scores.extend(list(peak_scores))\n        series_ids.extend([ser_id] * len(peaks))\n\n    return np.array(series_ids), np.array(times), np.array(scores)\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyperparams = {'k_dist': 265,\n 'log_base': 14,\n 'log_scale': 0.4790984168498803,\n 'curr_max_power': 0.7379097334221248,\n 'weight_coeff': 1.4190240522794364,\n 'score_thresh_coeff': 1.3935907064707698,\n 'convolution_kernel': [0.15, 0.7, 0.15],\n 'section_weight_method': 'logarithmic',\n 'preds_reduction_power': 4.2185511941820355,\n 'overlap_coeff': 0.6879294472420623,\n 'min_distance': 21}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids_onsets, onsets, scores_onsets = get_actual_preds(res_preds, res_series_ids, res_steps, 'onset', hyperparams)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids_wakeups, wakeups, scores_wakeups = get_actual_preds(res_preds, res_series_ids, res_steps, 'wakeup', hyperparams)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del res_preds, res_series_ids, res_steps\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onset_preds = pl.DataFrame().with_columns([pl.lit(series_ids_onsets).cast(pl.Utf8).alias('series_id'),\n                                           pl.lit(onsets).cast(pl.Int64).alias('step'),\n                                           pl.lit('onset').cast(pl.Utf8).alias('event'),\n                                           pl.lit(scores_onsets).cast(pl.Float32).alias('score')])\n\nwakeup_preds = pl.DataFrame().with_columns([pl.lit(series_ids_wakeups).cast(pl.Utf8).alias('series_id'),\n                                           pl.lit(wakeups).cast(pl.Int64).alias('step'),\n                                           pl.lit('wakeup').cast(pl.Utf8).alias('event'),\n                                           pl.lit(scores_wakeups).cast(pl.Float32).alias('score')])\n\ntest_preds_df = pl.concat([onset_preds, wakeup_preds]).sort(by=['series_id', 'step']).to_pandas()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del onset_preds, wakeup_preds\n_ = gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_df.insert(0, 'row_id', np.arange(len(test_preds_df)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -r /kaggle/working/*","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_preds_df['event'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_preds_df.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"print(\"Done\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}