{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":153365353,"sourceType":"kernelVersion"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Pre-setting\n- Target：classification of events\n    - 4 classes; Awake, Sleep, Onset, Wakeup\n- Features：enmo, anglez, shift_anglez, hour_sin, hour_cos\n- Training method: LightGBM\n- Validation：Hold-out\n    - train, valid = 221, 56\n- Evaluation\n    - AP: 0.017\n- Pre trained model: https://www.kaggle.com/code/coyamax/fork-of-cmi-train2-11?kernelSessionId=153365353\n\n- The following notebook only deals with inference process. ","metadata":{}},{"cell_type":"markdown","source":"### Remark\nMost of the implementations from preparing dataset to post-process come from the following discussions and related source code: \nhttps://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states/discussion/452940","metadata":{}},{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import math\nimport random\nimport sys\nimport time\nimport psutil\nimport shutil\nimport random\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nimport lightgbm as lgb # ver 3.3.2\nimport pickle\nimport gc\n\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Optional\nfrom tqdm import tqdm\nfrom torch.utils.data import Dataset\nfrom torchvision.transforms.functional import resize\nfrom sklearn.metrics import label_ranking_average_precision_score\nfrom scipy.signal import find_peaks","metadata":{"execution":{"iopub.status.busy":"2023-11-27T14:38:46.908149Z","iopub.execute_input":"2023-11-27T14:38:46.908784Z","iopub.status.idle":"2023-11-27T14:38:52.953854Z","shell.execute_reply.started":"2023-11-27T14:38:46.908735Z","shell.execute_reply":"2023-11-27T14:38:52.952545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config","metadata":{}},{"cell_type":"code","source":"# preprocess\n\npre_cfg={'processed_dir':'/kaggle/working/processed_data',\n         'data_dir':'/kaggle/input/child-mind-institute-detect-sleep-states',\n         'phase':'test',\n         }\n\nSERIES_SCHEMA = {\n    \"series_id\": pl.Utf8,\n    \"step\": pl.UInt32,\n    \"anglez\": pl.Float32,\n    \"enmo\": pl.Float32,\n}\n\n\nFEATURE_NAMES = [\n    \"anglez\",\n    \"enmo\",\n    \"hour_sin\",\n    \"hour_cos\",\n]\n\nANGLEZ_MEAN = -8.810476\nANGLEZ_STD = 35.521877\nENMO_MEAN = 0.041315\nENMO_STD = 0.101829\n\n# inference\ninf_cfg = {'duration':5760,\n           'upsample_rate':1,\n           'downsample_rate':1,\n           'features':[\"anglez\", \"enmo\", \"hour_sin\", \"hour_cos\"],\n           'added_features':[\"shift_anglez\"],\n           'model_path':'/kaggle/input/fork-of-cmi-train2-11/lgb_model_bl3.pkl',\n           'submission_dir':'/kaggle/working/',\n           'score_th':0.01,\n           'distance':5000,\n           }","metadata":{"_uuid":"2ab4f212-882c-46db-acb7-a2e6d322dc27","_cell_guid":"09af3306-2e32-47bb-8a6f-3091ef13000b","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess on Test Series","metadata":{}},{"cell_type":"code","source":"def to_coord(x: pl.Expr, max_: int, name: str) -> list[pl.Expr]:\n    rad = 2 * np.pi * (x % max_) / max_\n    x_sin = rad.sin()\n    x_cos = rad.cos()\n\n    return [x_sin.alias(f\"{name}_sin\"), x_cos.alias(f\"{name}_cos\")]\n\n\ndef deg_to_rad(x: pl.Expr) -> pl.Expr:\n    return np.pi / 180 * x\n\n\ndef add_feature(series_df: pl.DataFrame) -> pl.DataFrame:\n    series_df = (\n        series_df.with_row_count(\"step\")\n        .with_columns(\n            *to_coord(pl.col(\"timestamp\").dt.hour(), 24, \"hour\"),\n            *to_coord(pl.col(\"timestamp\").dt.month(), 12, \"month\"),\n            *to_coord(pl.col(\"timestamp\").dt.minute(), 60, \"minute\"),\n            pl.col(\"step\") / pl.count(\"step\"),\n            pl.col('anglez_rad').sin().alias('anglez_sin'),\n            pl.col('anglez_rad').cos().alias('anglez_cos'),\n        )\n        .select(\"series_id\", *FEATURE_NAMES)\n    )\n    return series_df\n\n\ndef save_each_series(this_series_df: pl.DataFrame, columns: list[str], output_dir: Path):\n    output_dir.mkdir(parents=True, exist_ok=True)\n\n    for col_name in columns:\n        x = this_series_df.get_column(col_name).to_numpy(zero_copy_only=True)\n        np.save(output_dir / f\"{col_name}.npy\", x)\n\ndef preprocess(cfg: pre_cfg):\n    processed_dir: Path = Path(cfg['processed_dir']) / cfg['phase']\n\n    # delete if directories already exist\n    if processed_dir.exists():\n        shutil.rmtree(processed_dir)\n        print(f\"Removed {cfg['phase']} dir: {processed_dir}\")\n\n    if cfg['phase'] in [\"train\", \"test\"]:\n        series_lf = pl.scan_parquet(\n            Path(cfg['data_dir']) / f\"{cfg['phase']}_series.parquet\",\n            low_memory=True,\n        )\n    else:\n        raise ValueError(f\"Invalid phase: {cfg['phase']}\")\n\n    # preprocess\n    series_df = (\n        series_lf.with_columns(\n            pl.col(\"timestamp\").str.strptime(pl.Datetime,format=\"%Y-%m-%dT%H:%M:%S%z\"),\n            deg_to_rad(pl.col(\"anglez\")).alias(\"anglez_rad\"),\n            (pl.col(\"anglez\") - ANGLEZ_MEAN) / ANGLEZ_STD,\n            (pl.col(\"enmo\") - ENMO_MEAN) / ENMO_STD,\n        )\n        .select(\n            [\n                pl.col(\"series_id\"),\n                pl.col(\"anglez\"),\n                pl.col(\"enmo\"),\n                pl.col(\"timestamp\"),\n                pl.col(\"anglez_rad\"),\n            ]\n        )\n        .collect(streaming=True)\n        .sort(by=[\"series_id\", \"timestamp\"])\n    )\n    n_unique = series_df.get_column(\"series_id\").n_unique()\n\n    for series_id, this_series_df in tqdm(series_df.groupby(\"series_id\"), total=n_unique):\n        this_series_df = add_feature(this_series_df)\n        series_dir = processed_dir / series_id  # type: ignore\n        save_each_series(this_series_df, FEATURE_NAMES, series_dir)\n\npreprocess(pre_cfg)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare test dataset","metadata":{}},{"cell_type":"code","source":"def pad_if_needed(x: np.ndarray, max_len: int, pad_value: float = 0.0) -> np.ndarray:\n    if len(x) == max_len:\n        return x\n    num_pad = max_len - len(x)\n    n_dim = len(x.shape)\n    pad_widths = [(0, num_pad)] + [(0, 0) for _ in range(n_dim - 1)]\n    return np.pad(x, pad_width=pad_widths, mode=\"constant\", constant_values=pad_value)\n\ndef load_chunk_features(\n    duration: int,\n    feature_names: list[str],\n    series_ids: Optional[list[str]],\n    processed_dir: Path,\n    phase: str,\n) -> dict[str, np.ndarray]:\n    features = {}\n\n    if series_ids is None:\n        series_ids = [series_dir.name for series_dir in (processed_dir / phase).glob(\"*\")]\n\n    for series_id in series_ids:\n        series_dir = processed_dir / phase / series_id\n        this_feature = []\n        for feature_name in feature_names:\n            this_feature.append(np.load(series_dir / f\"{feature_name}.npy\"))\n        this_feature = np.stack(this_feature, axis=1)\n        num_chunks = (len(this_feature) // duration) + 1\n        for i in range(num_chunks):\n            chunk_feature = this_feature[i * duration : (i + 1) * duration]\n            chunk_feature = pad_if_needed(chunk_feature, duration, pad_value=0)  # type: ignore\n            features[f\"{series_id}_{i:07}\"] = chunk_feature\n\n    return features  # type: ignore # shape(duration, features)\n\nfeature_names = inf_cfg['features']\nseries_ids = None\nprocessed_dir = Path(pre_cfg['processed_dir'])\nphase = pre_cfg['phase']\n\nchunk_features = load_chunk_features(duration=inf_cfg['duration'],\n                                     feature_names=feature_names,\n                                     series_ids=series_ids,\n                                     processed_dir=processed_dir,\n                                     phase=phase)\n\ndef nearest_valid_size(input_size: int, downsample_rate: int) -> int:\n    \"\"\"\n    return x which is closest to hop_length and satisties:\n    (x // hop_length) % 32 == 0\n    \"\"\"\n\n    while (input_size // downsample_rate) % 32 != 0:\n        input_size += 1\n    assert (input_size // downsample_rate) % 32 == 0\n\n    return input_size\n\nclass SegTestDataset(Dataset):\n    def __init__(\n        self,\n        cfg: dict,\n        chunk_features: dict[str, np.ndarray],\n    ):\n        self.cfg = cfg\n        self.chunk_features = chunk_features\n        self.keys = list(chunk_features.keys())\n        self.num_features = len(cfg['features'])\n        self.upsampled_num_frames = nearest_valid_size(\n            int(self.cfg['duration'] * self.cfg['upsample_rate']), self.cfg['downsample_rate']\n        )\n\n    def __len__(self):\n        return len(self.keys)\n\n    def __getitem__(self, idx):\n        key = self.keys[idx]\n        feature = self.chunk_features[key]\n        feature = torch.FloatTensor(feature.T).unsqueeze(0)  # (1, num_features, duration)\n        feature = resize(\n            feature,\n            size=[self.num_features, self.upsampled_num_frames],\n            antialias=False,\n        ).squeeze(0)\n\n        return {\n            \"key\": key,\n            \"feature\": feature,  # (num_features, duration)\n        }\n\ndef get_test_ds(\n    cfg,\n    chunk_features: dict[str, np.ndarray],):\n    return SegTestDataset(cfg=cfg, chunk_features=chunk_features)\n\ntest_ds = get_test_ds(cfg=inf_cfg, chunk_features=chunk_features)\n\n# save some memory\ndel chunk_features\ngc.collect();\n\n# add lagged features\ndef concat_ds(ds):\n    dfs = []\n    for i in range(ds.__len__()):\n        sample = ds[i]\n        series_id = sample[\"key\"]\n        feature = sample[\"feature\"]\n\n        sample['feature'] = feature.transpose(0,1).numpy()\n        sample[\"anglez\"] = sample['feature'][:,0]\n        sample[\"enmo\"] = sample['feature'][:,1]\n        sample[\"hour_sin\"] = sample['feature'][:,2]\n        sample[\"hour_cos\"] = sample['feature'][:,3]\n\n        del sample['feature']\n\n        sample_df = pl.DataFrame(sample)\n        lag = sample_df.to_pandas().groupby(['key'])[['anglez']].apply(lambda x: x.shift(1))\n        lag.columns=['shift_anglez']\n        lag = pl.from_pandas(lag)\n        sample_df = sample_df.with_columns(lag)\n        dfs.append(sample_df)\n    df = pl.concat(dfs)\n    \n    return df\n\ntest_ds = concat_ds(test_ds)\n\nfeature_names = inf_cfg['features'] + inf_cfg['added_features']","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load model and inference","metadata":{}},{"cell_type":"code","source":"with open(inf_cfg['model_path'], 'rb') as f:\n    gbm = pickle.load(f)\n\nkeys = test_ds.select('key').to_series().to_list()\n\nX_test = test_ds.select(feature_names)\ny_test = gbm.predict(X_test)\npreds = y_test[:,2:4] # onset and wakeup","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## make submission file","metadata":{}},{"cell_type":"code","source":"def post_process_for_seg(\n    keys: list[str], preds: np.ndarray, score_th: float = 0.01, distance: int = 5000\n) -> pl.DataFrame:\n    \"\"\"make submission dataframe for segmentation task\n\n    Args:\n        keys (list[str]): list of keys. key is \"{series_id}_{chunk_id}\"\n        preds (np.ndarray): (num_series * num_chunks, duration, 2)\n        score_th (float, optional): threshold for score.\n\n    Returns:\n        pl.DataFrame: submission dataframe\n    \"\"\"\n    series_ids = np.array(list(map(lambda x: x.split(\"_\")[0], keys)))\n    unique_series_ids = np.unique(series_ids)\n\n    records = []\n    for series_id in unique_series_ids:\n        series_idx = np.where(series_ids == series_id)[0]\n        this_series_preds = preds[series_idx].reshape(-1, 2)\n\n        for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n            this_event_preds = this_series_preds[:, i]\n            steps = find_peaks(this_event_preds, height=score_th, distance=distance)[0]\n            scores = this_event_preds[steps]\n\n            for step, score in zip(steps, scores):\n                records.append(\n                    {\n                        \"series_id\": series_id,\n                        \"step\": step,\n                        \"event\": event_name,\n                        \"score\": score,\n                    }\n                )\n\n    if len(records) == 0:  # put dummies if no prediction \n        records.append(\n            {\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            }\n        )\n\n    sub_df = pl.DataFrame(records).sort(by=[\"series_id\", \"step\"])\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df\n\ndef make_submission(\n    keys: list[str], preds: np.ndarray, score_th, distance\n) -> pl.DataFrame:\n    sub_df = post_process_for_seg(\n        keys,\n        preds,  # type: ignore\n        score_th=score_th,\n        distance=distance,  # type: ignore\n    )\n\n    return sub_df\n\nsub_df = make_submission(\n    keys,\n    preds,\n    score_th=inf_cfg['score_th'],\n    distance=inf_cfg['distance'],\n)\nsub_df.write_csv(Path(inf_cfg['submission_dir']) / \"submission.csv\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# delete unnecessary folders\n!rm -rf /kaggle/working/processed_data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### END","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}