{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7104787,"sourceType":"datasetVersion","datasetId":3960176},{"sourceId":7127705,"sourceType":"datasetVersion","datasetId":3986324},{"sourceId":144358840,"sourceType":"kernelVersion"},{"sourceId":150019889,"sourceType":"kernelVersion"},{"sourceId":152708379,"sourceType":"kernelVersion"},{"sourceId":152712639,"sourceType":"kernelVersion"},{"sourceId":153270887,"sourceType":"kernelVersion"},{"sourceId":153280254,"sourceType":"kernelVersion"},{"sourceId":153295927,"sourceType":"kernelVersion"},{"sourceId":153590541,"sourceType":"kernelVersion"},{"sourceId":153594440,"sourceType":"kernelVersion"},{"sourceId":153704820,"sourceType":"kernelVersion"},{"sourceId":153722311,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 共通パート","metadata":{}},{"cell_type":"code","source":"!mkdir /kaggle/temp/","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:01.635688Z","iopub.execute_input":"2023-12-05T10:48:01.636014Z","iopub.status.idle":"2023-12-05T10:48:02.693627Z","shell.execute_reply.started":"2023-12-05T10:48:01.635987Z","shell.execute_reply":"2023-12-05T10:48:02.692163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile settings.py\n\nfrom pathlib import Path\n\ndebug = False\ndebug_sample_size = 10\nmode = \"train\" if debug else \"test\"\n\noutput_dir = Path(\"/kaggle/temp/sakami-output\")\noutput_dir.mkdir(exist_ok=True)\n\nshimacos_output_dir = Path(\"/kaggle/temp/shimacos-output\")\nshimacos_output_dir.mkdir(exist_ok=True)\n\nsakami_model_names_009 = [\n \"148_gru_scale_factor\",\n \"156_gru_transformer_residual\",\n \"163_gru_sleep_target\",\n]\nkami_model_names_009 = [\"exp068_transformer\", \"exp078_lstm\"]\n\nsakami_model_names_030 = [\n \"148_gru_scale_factor\",\n \"156_gru_transformer_residual\",\n \"163_gru_sleep_target\",\n \"179_gru_minute_embedding_sync\",\n]\nkami_model_names_030 = [\"exp068_transformer\", \"exp078_lstm\", \"exp081_mixup_short_feat14\"]\n\ndata_chunk_size = 2\n\nperiodicity_dir = Path(f\"/kaggle/temp/processed_data/test/\")\ndata_dir1 = Path(\"/kaggle/input/sakami-148-gru-scale-factor\")\ndata_dir2 = Path(\"/kaggle/input/sakami-156-gru-transformer-residual\")\ndata_dir3 = Path(\"/kaggle/input/sakami-163-gru-sleep-target\")\ndata_dir4 = Path(\"/kaggle/input/sakami-179-gru-minute-embedding-sync\")\nstacking_dir1 = Path(\"/kaggle/input/sakami-stacking-004-transformer-category-padding\")\nstacking_dir2 = Path(\"/kaggle/input/sakami-stacking-011-cnn-embedding-sync\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:08:22.341093Z","iopub.execute_input":"2023-12-05T11:08:22.341473Z","iopub.status.idle":"2023-12-05T11:08:22.347813Z","shell.execute_reply.started":"2023-12-05T11:08:22.34143Z","shell.execute_reply":"2023-12-05T11:08:22.346908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prepare periodicity data","metadata":{}},{"cell_type":"code","source":"!pip install -q hydra-core segmentation_models_pytorch==0.3.3 --no-index --find-links=/kaggle/input/ex-library","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:12.739162Z","iopub.execute_input":"2023-12-05T10:48:12.740082Z","iopub.status.idle":"2023-12-05T10:48:33.511722Z","shell.execute_reply.started":"2023-12-05T10:48:12.740041Z","shell.execute_reply":"2023-12-05T10:48:33.51065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%cd /kaggle/input/cmi-code/kaggle-child-mind-institute-detect-sleep-states-master","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.513627Z","iopub.execute_input":"2023-12-05T10:48:33.513898Z","iopub.status.idle":"2023-12-05T10:48:33.518399Z","shell.execute_reply.started":"2023-12-05T10:48:33.513875Z","shell.execute_reply":"2023-12-05T10:48:33.517497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python -m run.prepare_data dir=kaggle dir.processed_dir=\"/kaggle/temp/processed_data/\" phase=$MODE","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.519681Z","iopub.execute_input":"2023-12-05T10:48:33.519939Z","iopub.status.idle":"2023-12-05T10:48:33.528965Z","shell.execute_reply.started":"2023-12-05T10:48:33.519916Z","shell.execute_reply":"2023-12-05T10:48:33.528169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.530937Z","iopub.execute_input":"2023-12-05T10:48:33.531209Z","iopub.status.idle":"2023-12-05T10:48:33.537914Z","shell.execute_reply.started":"2023-12-05T10:48:33.531179Z","shell.execute_reply":"2023-12-05T10:48:33.537118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!ls /kaggle/temp/processed_data/$MODE/","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.539107Z","iopub.execute_input":"2023-12-05T10:48:33.540057Z","iopub.status.idle":"2023-12-05T10:48:33.5472Z","shell.execute_reply.started":"2023-12-05T10:48:33.540028Z","shell.execute_reply":"2023-12-05T10:48:33.546363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KAMI\n","metadata":{}},{"cell_type":"code","source":"#PHASE = os.environ[\"MODE\"]\nPROCESSED_DIR = '/kaggle/temp/processed_data'\nOUTPUT_DIR = '/kaggle/temp/kami_output'\nDATA_DIR = \"/kaggle/temp/kami_data\"","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.548051Z","iopub.execute_input":"2023-12-05T10:48:33.548302Z","iopub.status.idle":"2023-12-05T10:48:33.556658Z","shell.execute_reply.started":"2023-12-05T10:48:33.54828Z","shell.execute_reply":"2023-12-05T10:48:33.555792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## prepare data","metadata":{}},{"cell_type":"code","source":"!mkdir $DATA_DIR","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:33.557839Z","iopub.execute_input":"2023-12-05T10:48:33.558165Z","iopub.status.idle":"2023-12-05T10:48:34.505236Z","shell.execute_reply.started":"2023-12-05T10:48:33.558117Z","shell.execute_reply":"2023-12-05T10:48:34.503865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cp /kaggle/input/child-mind-institute-detect-sleep-states/* $DATA_DIR","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:34.506756Z","iopub.execute_input":"2023-12-05T10:48:34.507037Z","iopub.status.idle":"2023-12-05T10:48:43.765241Z","shell.execute_reply.started":"2023-12-05T10:48:34.507012Z","shell.execute_reply":"2023-12-05T10:48:43.764019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nfrom pathlib import Path\n\nimport polars as pl\n\nfrom settings import debug, debug_sample_size, output_dir\n\ninput_dir = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states/\")\nkami_output_dir = Path(\"/kaggle/temp/kami_data\")\n\nif debug:\n test_df = pl.read_parquet(input_dir / \"train_series.parquet\")\n test_series = test_df[\"series_id\"].unique(maintain_order=True).sample(debug_sample_size, seed=1029)\n test_df = test_df.filter(pl.col(\"series_id\").is_in(test_series)).sort([\"series_id\", \"step\"])\n test_df.write_parquet(output_dir / \"test_series.parquet\")\n test_df.write_parquet(kami_output_dir / \"test_series.parquet\")\nelse:\n test_df = pl.read_parquet(input_dir / \"test_series.parquet\").sort([\"series_id\", \"step\"])\n test_df.write_parquet(output_dir / \"test_series.parquet\")\n test_df.write_parquet(kami_output_dir / \"test_series.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:48:43.76685Z","iopub.execute_input":"2023-12-05T10:48:43.767226Z","iopub.status.idle":"2023-12-05T10:49:05.60958Z","shell.execute_reply.started":"2023-12-05T10:48:43.767191Z","shell.execute_reply":"2023-12-05T10:49:05.608775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## inference","metadata":{}},{"cell_type":"code","source":"%cd /kaggle/input/cmi-code/kaggle-child-mind-institute-detect-sleep-states-master","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:49:05.612977Z","iopub.execute_input":"2023-12-05T10:49:05.613246Z","iopub.status.idle":"2023-12-05T10:49:05.62361Z","shell.execute_reply.started":"2023-12-05T10:49:05.613222Z","shell.execute_reply":"2023-12-05T10:49:05.622577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m run.prepare_data dir=kaggle dir.processed_dir=\"/kaggle/temp/processed_data/\" dir.data_dir=$DATA_DIR phase=test","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:49:05.624912Z","iopub.execute_input":"2023-12-05T10:49:05.625248Z","iopub.status.idle":"2023-12-05T10:49:42.750179Z","shell.execute_reply.started":"2023-12-05T10:49:05.625209Z","shell.execute_reply":"2023-12-05T10:49:42.749069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exp081_mixup_short_feat14\n!python -m run.cv_inference\\\n phase=test\\\n dir=kaggle\\\n dir.data_dir=$DATA_DIR dir.cv_model_dir=/kaggle/input/cmi-cv-model dir.processed_dir=$PROCESSED_DIR dir.output_dir=$OUTPUT_DIR\\\n exp_name=exp081_mixup_short_feat14 model.encoder_weights=null model=Spec2DCNNSplit model.n_split=1 datamodule.zero_periodicity=True duration=17280 batch_size=8 \"features=014\" feature_extractor=LSTMFeatureExtractor downsample_rate=6 num_tta=3\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:49:42.75172Z","iopub.execute_input":"2023-12-05T10:49:42.752043Z","iopub.status.idle":"2023-12-05T10:52:34.119587Z","shell.execute_reply.started":"2023-12-05T10:49:42.752015Z","shell.execute_reply":"2023-12-05T10:52:34.118418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exp078_lstm\n!python -m run.cv_inference\\\n phase=test\\\n dir=kaggle\\\n dir.data_dir=$DATA_DIR dir.cv_model_dir=/kaggle/input/cmi-cv-model dir.processed_dir=$PROCESSED_DIR dir.output_dir=$OUTPUT_DIR\\\n exp_name=exp078_lstm model.encoder_weights=null model=Spec2DCNNSplit model.n_split=1 datamodule.zero_periodicity=True duration=17280 batch_size=8 \"features=012\" feature_extractor=LSTMFeatureExtractor num_tta=3","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:52:34.120916Z","iopub.execute_input":"2023-12-05T10:52:34.12119Z","iopub.status.idle":"2023-12-05T10:57:05.401884Z","shell.execute_reply.started":"2023-12-05T10:52:34.121165Z","shell.execute_reply":"2023-12-05T10:57:05.400768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# exp068_transformer\n\n!python -m run.cv_inference\\\n phase=test\\\n dir=kaggle\\\n dir.data_dir=$DATA_DIR dir.cv_model_dir=/kaggle/input/cmi-cv-model dir.processed_dir=$PROCESSED_DIR dir.output_dir=$OUTPUT_DIR\\\n exp_name=exp068_transformer model.encoder_weights=null model=Spec2DCNNSplit model.n_split=1 duration=17280 batch_size=8 \"features=012\" num_tta=3 decoder=TransformerDecoder downsample_rate=2 decoder.num_layers=3","metadata":{"execution":{"iopub.status.busy":"2023-12-05T10:57:05.403307Z","iopub.execute_input":"2023-12-05T10:57:05.403613Z","iopub.status.idle":"2023-12-05T11:02:03.423913Z","shell.execute_reply.started":"2023-12-05T10:57:05.403586Z","shell.execute_reply":"2023-12-05T11:02:03.422942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 出力先","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/temp/kami_output/cv_inference/exp081_mixup_short_feat14/single","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:03.425402Z","iopub.execute_input":"2023-12-05T11:02:03.425789Z","iopub.status.idle":"2023-12-05T11:02:04.381184Z","shell.execute_reply.started":"2023-12-05T11:02:03.425752Z","shell.execute_reply":"2023-12-05T11:02:04.38006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/temp/kami_output/cv_inference/exp068_transformer/single","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:04.38272Z","iopub.execute_input":"2023-12-05T11:02:04.38307Z","iopub.status.idle":"2023-12-05T11:02:05.315674Z","shell.execute_reply.started":"2023-12-05T11:02:04.383041Z","shell.execute_reply":"2023-12-05T11:02:05.314501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/temp/kami_output/cv_inference/exp078_lstm/single","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:05.317151Z","iopub.execute_input":"2023-12-05T11:02:05.317471Z","iopub.status.idle":"2023-12-05T11:02:06.250442Z","shell.execute_reply.started":"2023-12-05T11:02:05.317429Z","shell.execute_reply":"2023-12-05T11:02:06.249492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /kaggle/working/","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:06.251877Z","iopub.execute_input":"2023-12-05T11:02:06.252186Z","iopub.status.idle":"2023-12-05T11:02:06.258888Z","shell.execute_reply.started":"2023-12-05T11:02:06.252159Z","shell.execute_reply":"2023-12-05T11:02:06.257925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SAKAMI","metadata":{}},{"cell_type":"markdown","source":"## Inference","metadata":{}},{"cell_type":"code","source":"!pip install -q --no-deps \\\n /kaggle/input/sakami-pip-wheels/google_cloud_storage-2.9.0-py2.py3-none-any.whl \\\n /kaggle/input/sakami-pip-wheels/invoke-2.2.0-py3-none-any.whl \\\n /kaggle/input/sakami-pip-wheels/loguru-0.7.0-py3-none-any.whl \\\n /kaggle/input/sakami-pip-wheels/nbparameterise-0.6-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:06.260168Z","iopub.execute_input":"2023-12-05T11:02:06.260533Z","iopub.status.idle":"2023-12-05T11:02:36.769091Z","shell.execute_reply.started":"2023-12-05T11:02:06.2605Z","shell.execute_reply":"2023-12-05T11:02:36.76792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cp -r /kaggle/input/sakami-src-files/src/ .","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:36.770494Z","iopub.execute_input":"2023-12-05T11:02:36.770791Z","iopub.status.idle":"2023-12-05T11:02:37.746075Z","shell.execute_reply.started":"2023-12-05T11:02:36.770764Z","shell.execute_reply":"2023-12-05T11:02:37.744789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile utils.py\n\nimport time\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef timer(message: str) -> None:\n print(f\"[{message}] start.\")\n start_time = time.time()\n yield\n elapsed_time = (time.time() - start_time) / 60\n print(f\"[{message}] done in {elapsed_time:.1f} min.\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:37.747733Z","iopub.execute_input":"2023-12-05T11:02:37.748503Z","iopub.status.idle":"2023-12-05T11:02:37.755229Z","shell.execute_reply.started":"2023-12-05T11:02:37.748441Z","shell.execute_reply":"2023-12-05T11:02:37.754362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nfrom pathlib import Path\n\nimport numpy as np\nimport polars as pl\n\nfrom settings import output_dir, periodicity_dir\nfrom utils import timer\n\nwith timer(\"load data\"):\n test_df = pl.read_parquet(output_dir / \"test_series.parquet\")[[\"series_id\", \"step\"]]\n\nwith timer(\"add periodicity\"):\n # add periodicity\n periodicity_series_ids = []\n periodicity_flags = []\n periodicity_steps = []\n for series_dir in periodicity_dir.iterdir():\n periodicity_flag = np.load(series_dir / \"periodicity.npy\")\n periodicity_series_ids.append(series_dir.stem)\n periodicity_flags.append(periodicity_flag)\n periodicity_steps.append(np.arange(len(periodicity_flag)))\n\n periodicity_df = (\n pl.DataFrame(\n {\n \"series_id\": periodicity_series_ids,\n \"step\": periodicity_steps,\n \"periodicity\": periodicity_flags,\n }\n )\n .explode(columns=[\"step\", \"periodicity\"])\n .with_columns(pl.col(\"step\").cast(pl.UInt32), pl.col(\"periodicity\") + 1)\n )\n test_df = test_df.join(periodicity_df, how=\"left\", on=[\"series_id\", \"step\"])\n\nnp.save(output_dir / \"periodicity.npy\", test_df[\"periodicity\"].to_numpy())","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:37.756586Z","iopub.execute_input":"2023-12-05T11:02:37.756883Z","iopub.status.idle":"2023-12-05T11:02:40.75022Z","shell.execute_reply.started":"2023-12-05T11:02:37.756858Z","shell.execute_reply":"2023-12-05T11:02:40.749214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nfrom __future__ import annotations\n\nimport gc\nimport itertools\nfrom pathlib import Path\nfrom runpy import run_path\n\nimport joblib\nimport numpy as np\nimport polars as pl\nimport torch\nfrom torch.utils.data import DataLoader\n\nfrom settings import data_dir1, output_dir, periodicity_dir\nfrom utils import timer\n\nmodule = run_path(str(next(data_dir1.glob(\"*.py\"))))\nconfig = module[\"config\"]\n\nwith timer(\"load data\"):\n test_df = pl.read_parquet(output_dir / \"test_series.parquet\").with_columns(\n pl.col([\"anglez\", \"enmo\"]).cast(pl.Float32)\n )\n\nwith timer(\"feature engineering\"):\n test_df = test_df.with_columns(\n pl.col(\"anglez\") / 45,\n (pl.col(\"enmo\").log1p() / 0.1).clip_max(10.0),\n pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\"),\n ).with_columns(\n pl.col(\"timestamp\").dt.hour().alias(\"hour\") + 1,\n pl.col(\"timestamp\").dt.minute().alias(\"minute\") + 1,\n pl.col(\"timestamp\").dt.weekday().alias(\"weekday\"),\n pl.col([\"anglez\", \"enmo\"]).rolling_mean(12, center=True).over(\"series_id\").suffix(\"_12_rolling_mean\"),\n pl.col([\"anglez\", \"enmo\"]).rolling_std(12, center=True).over(\"series_id\").suffix(\"_12_rolling_std\"),\n pl.col([\"anglez\", \"enmo\"]).rolling_max(12, center=True).over(\"series_id\").suffix(\"_12_rolling_max\"),\n pl.col(\"anglez\").diff().abs().rolling_median(60, center=True).over(\"series_id\").suffix(\"_diff_5min_median\"),\n )\n\n periodicity_flags = np.load(output_dir / \"periodicity.npy\")\n test_df = test_df.with_columns(pl.Series(\"periodicity\", periodicity_flags))\n\n categorical_columns = [\"hour\", \"minute\", \"weekday\", \"periodicity\"]\n numerical_columns = test_df.select(r\"^(anglez|enmo).*$\").columns\n\n numerical_inputs = test_df.select(numerical_columns).fill_null(0.0).to_numpy().astype(np.float32)\n categorical_inputs = test_df.select(categorical_columns).fill_null(0.0).to_numpy().astype(np.int64)\n\n # pad inputs\n numerical_inputs = np.pad(numerical_inputs, ((0, 1), (0, 0)))\n categorical_inputs = np.pad(categorical_inputs, ((0, 1), (0, 0)))\n\n np.save(output_dir / \"numerical_inputs.npy\", numerical_inputs)\n np.save(output_dir / \"categorical_inputs.npy\", categorical_inputs)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:02:40.751718Z","iopub.execute_input":"2023-12-05T11:02:40.752394Z","iopub.status.idle":"2023-12-05T11:03:00.548785Z","shell.execute_reply.started":"2023-12-05T11:02:40.752358Z","shell.execute_reply":"2023-12-05T11:03:00.547917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport numpy as np\nimport polars as pl\n\nfrom settings import data_chunk_size, output_dir\nfrom utils import timer\n\nwith timer(\"load data\"):\n test_df = pl.read_parquet(output_dir / \"test_series.parquet\").with_columns(\n pl.arange(0, pl.count()).alias(\"row_id\")\n )\n numerical_inputs = np.load(output_dir / \"numerical_inputs.npy\")\n categorical_inputs = np.load(output_dir / \"categorical_inputs.npy\")\n\nwith timer(\"split data\"):\n series_ids = test_df[\"series_id\"].unique()\n chunk_series_ids_list = np.array_split(series_ids, data_chunk_size)\n for i, chunk_series_ids in enumerate(chunk_series_ids_list):\n chunk_test_df = test_df.filter(pl.col(\"series_id\").is_in(chunk_series_ids))\n chunk_test_df.write_parquet(output_dir / f\"test_series_{i}.parquet\")\n\n np.save(output_dir / f\"numerical_inputs_{i}.npy\", np.pad(numerical_inputs[chunk_test_df[\"row_id\"]], ((0, 1), (0, 0))))\n np.save(output_dir / f\"categorical_inputs_{i}.npy\", np.pad(categorical_inputs[chunk_test_df[\"row_id\"]], ((0, 1), (0, 0))))","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:03:00.550128Z","iopub.execute_input":"2023-12-05T11:03:00.550688Z","iopub.status.idle":"2023-12-05T11:03:04.150914Z","shell.execute_reply.started":"2023-12-05T11:03:00.550651Z","shell.execute_reply":"2023-12-05T11:03:04.150066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nfrom __future__ import annotations\n\nimport gc\nimport itertools\nfrom pathlib import Path\nfrom runpy import run_path\n\nimport joblib\nimport numpy as np\nimport polars as pl\nimport torch\nfrom torch.utils.data import DataLoader\n\nfrom settings import data_chunk_size, data_dir1, data_dir2, data_dir3, data_dir4, output_dir, periodicity_dir\nfrom utils import timer\n\nmodel_dir1 = data_dir1 / \"models\"\nmodel_dir2 = data_dir2 / \"models\"\nmodel_dir3 = data_dir3 / \"models\"\nmodel_dir4 = data_dir4 / \"models\"\n\ndevice = \"cuda\"\nbatch_size = 32\n\nmodule1 = run_path(str(next(data_dir1.glob(\"*.py\"))))\nconfig1 = module1[\"config\"]\nSleepDataset1 = module1[\"SleepDataset\"]\nSleepModel1 = module1[\"SleepModel\"]\npredict1 = module1[\"predict\"]\n\nmodule2 = run_path(str(next(data_dir2.glob(\"*.py\"))))\nconfig2 = module2[\"config\"]\nSleepDataset2 = module2[\"SleepDataset\"]\nSleepModel2 = module2[\"SleepModel\"]\npredict2 = module2[\"predict\"]\n\nmodule3 = run_path(str(next(data_dir3.glob(\"*.py\"))))\nconfig3 = module3[\"config\"]\nSleepDataset3 = module3[\"SleepDataset\"]\nSleepModel3 = module3[\"SleepModel\"]\npredict3 = module3[\"predict\"]\n\nmodule4 = run_path(str(next(data_dir4.glob(\"*.py\"))))\nconfig4 = module4[\"config\"]\nSleepDataset4 = module4[\"SleepDataset\"]\nSleepModel4 = module4[\"SleepModel\"]\npredict4 = module4[\"predict\"]\n\nfor data_chunk_index in range(data_chunk_size):\n print(\"-\" * 40)\n print(f\"******* chunk {data_chunk_index} *******\")\n\n with timer(\"load data\"):\n test_df = pl.read_parquet(output_dir / f\"test_series_{data_chunk_index}.parquet\", columns=[\"series_id\", \"row_id\"])\n numerical_inputs = np.load(output_dir / f\"numerical_inputs_{data_chunk_index}.npy\")\n categorical_inputs = np.load(output_dir / f\"categorical_inputs_{data_chunk_index}.npy\")\n\n with timer(\"split into chunks\"):\n series_row_ids = dict(\n test_df.with_columns(pl.arange(0, pl.count()).alias(\"index\"))\n .group_by(\"series_id\").agg(\"index\").rows()\n )\n\n series_chunk_ids = [] # list[str]\n series_chunk_row_ids = [] # list[list[int]]\n for series_id, row_ids in series_row_ids.items():\n for start_idx in range(\n -config1.preds_truncate_size, len(row_ids), int(config1.stride_size / config1.epoch_sample_rate)\n ):\n if start_idx + config1.chunk_size <= len(row_ids) + config1.preds_truncate_size:\n chunk_row_ids = row_ids[max(0, start_idx) : start_idx + config1.chunk_size]\n\n # padding\n if len(chunk_row_ids) < config1.chunk_size:\n chunk_row_ids = [-1] * (config1.chunk_size - len(chunk_row_ids)) + chunk_row_ids\n\n series_chunk_ids.append(series_id)\n series_chunk_row_ids.append(np.array(chunk_row_ids))\n else:\n chunk_row_ids = row_ids[-config1.chunk_size + config1.preds_truncate_size :]\n\n # padding\n if len(chunk_row_ids) < config1.chunk_size:\n chunk_row_ids = chunk_row_ids + [-1] * (config1.chunk_size - len(chunk_row_ids))\n\n series_chunk_ids.append(series_id)\n series_chunk_row_ids.append(np.array(chunk_row_ids))\n break\n\n del series_row_ids\n gc.collect()\n\n with timer(\"predict model 1\"):\n targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n test_dataset = SleepDataset1(\n series_chunk_row_ids=series_chunk_row_ids,\n numerical_inputs=numerical_inputs,\n categorical_inputs=categorical_inputs,\n targets=targets,\n max_series_length=config1.chunk_size,\n )\n test_loader = DataLoader(\n test_dataset,\n batch_size=batch_size,\n shuffle=False,\n drop_last=False,\n num_workers=4,\n pin_memory=True,\n )\n\n model = SleepModel1(\n numerical_input_size=numerical_inputs.shape[1],\n categorical_input_size=categorical_inputs.shape[1],\n )\n model.eval()\n model.to(device)\n\n prediction_values = np.zeros((len(numerical_inputs) - 1, 2), dtype=np.float32)\n model_paths = list(model_dir1.glob(\"model_fold*.pth\"))\n for model_path in model_paths:\n model.load_state_dict(torch.load(model_path, map_location=device))\n pred_df = predict1(model, test_loader, config1.preds_truncate_size).filter(pl.col(\"row_id\") != -1)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 0] += pred_df[\"prediction_onset\"].to_numpy() / len(model_paths)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 1] += pred_df[\"prediction_wakeup\"].to_numpy() / len(model_paths)\n\n del test_dataset, test_loader\n gc.collect()\n\n pl.DataFrame(\n {\n \"row_id\": test_df[\"row_id\"],\n \"prediction_onset\": prediction_values[:, 0],\n \"prediction_wakeup\": prediction_values[:, 1],\n }\n ).write_parquet(output_dir / f\"148_test_preds_{data_chunk_index}.parquet\")\n\n with timer(\"predict model 2\"):\n targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n test_dataset = SleepDataset2(\n series_chunk_row_ids=series_chunk_row_ids,\n numerical_inputs=numerical_inputs,\n categorical_inputs=categorical_inputs,\n targets=targets,\n max_series_length=config2.chunk_size,\n )\n test_loader = DataLoader(\n test_dataset,\n batch_size=batch_size,\n shuffle=False,\n drop_last=False,\n num_workers=4,\n pin_memory=True,\n )\n\n model = SleepModel2(\n model_name=data_dir2 / \"longformer-base-4096\",\n numerical_input_size=numerical_inputs.shape[1],\n categorical_input_size=categorical_inputs.shape[1],\n )\n model.eval()\n model.to(device)\n\n prediction_values = np.zeros((len(numerical_inputs) - 1, 2), dtype=np.float32)\n model_paths = list(model_dir2.glob(\"model_fold*.pth\"))\n for model_path in model_paths:\n model.load_state_dict(torch.load(model_path, map_location=device))\n pred_df = predict2(model, test_loader, config2.preds_truncate_size).filter(pl.col(\"row_id\") != -1)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 0] += pred_df[\"prediction_onset\"].to_numpy() / len(model_paths)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 1] += pred_df[\"prediction_wakeup\"].to_numpy() / len(model_paths)\n\n del test_dataset, test_loader\n gc.collect()\n\n pl.DataFrame(\n {\n \"row_id\": test_df[\"row_id\"],\n \"prediction_onset\": prediction_values[:, 0],\n \"prediction_wakeup\": prediction_values[:, 1],\n }\n ).write_parquet(output_dir / f\"156_test_preds_{data_chunk_index}.parquet\")\n\n with timer(\"predict model 3\"):\n targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n test_dataset = SleepDataset3(\n series_chunk_row_ids=series_chunk_row_ids,\n numerical_inputs=numerical_inputs,\n categorical_inputs=categorical_inputs,\n targets=targets,\n max_series_length=config3.chunk_size,\n )\n test_loader = DataLoader(\n test_dataset,\n batch_size=batch_size,\n shuffle=False,\n drop_last=False,\n num_workers=4,\n pin_memory=True,\n )\n\n model = SleepModel3(\n numerical_input_size=numerical_inputs.shape[1],\n categorical_input_size=categorical_inputs.shape[1],\n )\n model.eval()\n model.to(device)\n\n prediction_values = np.zeros((len(numerical_inputs) - 1, 3), dtype=np.float32)\n model_paths = list(model_dir3.glob(\"model_fold*.pth\"))\n for model_path in model_paths:\n model.load_state_dict(torch.load(model_path, map_location=device))\n pred_df = predict3(model, test_loader, config3.preds_truncate_size).filter(pl.col(\"row_id\") != -1)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 0] += pred_df[\"prediction_onset\"].to_numpy() / len(model_paths)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 1] += pred_df[\"prediction_wakeup\"].to_numpy() / len(model_paths)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 2] += pred_df[\"sleep\"].to_numpy() / len(model_paths)\n\n del test_dataset, test_loader\n gc.collect()\n\n pl.DataFrame(\n {\n \"row_id\": test_df[\"row_id\"],\n \"prediction_onset\": prediction_values[:, 0],\n \"prediction_wakeup\": prediction_values[:, 1],\n \"sleep\": prediction_values[:, 2],\n }\n ).write_parquet(output_dir / f\"163_test_preds_{data_chunk_index}.parquet\")\n\n with timer(\"predict model 4\"):\n targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n test_dataset = SleepDataset4(\n series_chunk_row_ids=series_chunk_row_ids,\n numerical_inputs=numerical_inputs,\n categorical_inputs=categorical_inputs[:, [1, 0, 2, 3]],\n targets=targets,\n max_series_length=config4.chunk_size,\n )\n test_loader = DataLoader(\n test_dataset,\n batch_size=batch_size,\n shuffle=False,\n drop_last=False,\n num_workers=4,\n pin_memory=True,\n )\n\n model = SleepModel4(\n numerical_input_size=numerical_inputs.shape[1],\n categorical_input_size=categorical_inputs.shape[1],\n )\n model.eval()\n model.to(device)\n\n prediction_values = np.zeros((len(numerical_inputs) - 1, 2), dtype=np.float32)\n model_paths = list(model_dir4.glob(\"model_fold*.pth\"))\n for model_path in model_paths:\n model.load_state_dict(torch.load(model_path, map_location=device))\n pred_df = predict4(model, test_loader, config4.preds_truncate_size).filter(pl.col(\"row_id\") != -1)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 0] += pred_df[\"prediction_onset\"].to_numpy() / len(model_paths)\n prediction_values[pred_df[\"row_id\"].to_numpy(), 1] += pred_df[\"prediction_wakeup\"].to_numpy() / len(model_paths)\n\n del test_dataset, test_loader\n gc.collect()\n\n pl.DataFrame(\n {\n \"row_id\": test_df[\"row_id\"],\n \"prediction_onset\": prediction_values[:, 0],\n \"prediction_wakeup\": prediction_values[:, 1],\n }\n ).write_parquet(output_dir / f\"179_test_preds_{data_chunk_index}.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:03:04.152203Z","iopub.execute_input":"2023-12-05T11:03:04.152529Z","iopub.status.idle":"2023-12-05T11:07:36.260966Z","shell.execute_reply.started":"2023-12-05T11:03:04.1525Z","shell.execute_reply":"2023-12-05T11:07:36.260136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport os\nimport polars as pl\n\nfrom settings import data_chunk_size, output_dir\nfrom utils import timer\n\nwith timer(\"merge preds\"):\n preds = [\n pl.read_parquet(output_dir / f\"148_test_preds_{data_chunk_index}.parquet\")\n for data_chunk_index in range(data_chunk_size)\n ]\n preds_df = pl.concat(preds).sort(\"row_id\")\n os.makedirs(output_dir / '148_gru_scale_factor', exist_ok=True)\n preds_df.write_parquet(output_dir / \"148_gru_scale_factor/test_preds.parquet\")\n\n preds = [\n pl.read_parquet(output_dir / f\"156_test_preds_{data_chunk_index}.parquet\")\n for data_chunk_index in range(data_chunk_size)\n ]\n preds_df = pl.concat(preds).sort(\"row_id\")\n os.makedirs(output_dir / '156_gru_transformer_residual', exist_ok=True)\n preds_df.write_parquet(output_dir / \"156_gru_transformer_residual/test_preds.parquet\")\n\n preds = [\n pl.read_parquet(output_dir / f\"163_test_preds_{data_chunk_index}.parquet\")\n for data_chunk_index in range(data_chunk_size)\n ]\n preds_df = pl.concat(preds).sort(\"row_id\")\n os.makedirs(output_dir / '163_gru_sleep_target', exist_ok=True)\n preds_df.write_parquet(output_dir / \"163_gru_sleep_target/test_preds.parquet\")\n\n preds = [\n pl.read_parquet(output_dir / f\"179_test_preds_{data_chunk_index}.parquet\")\n for data_chunk_index in range(data_chunk_size)\n ]\n preds_df = pl.concat(preds).sort(\"row_id\")\n os.makedirs(output_dir / '179_gru_minute_embedding_sync', exist_ok=True)\n preds_df.write_parquet(output_dir / \"179_gru_minute_embedding_sync/test_preds.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:07:36.262095Z","iopub.execute_input":"2023-12-05T11:07:36.262393Z","iopub.status.idle":"2023-12-05T11:07:40.107775Z","shell.execute_reply.started":"2023-12-05T11:07:36.262368Z","shell.execute_reply":"2023-12-05T11:07:40.106746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%python\n\n# from pathlib import Path\n# from runpy import run_path\n\n# import numpy as np\n# import polars as pl\n# from scipy.ndimage import maximum_filter1d, uniform_filter1d\n# from scipy.signal import find_peaks\n\n# from settings import data_dir1, output_dir, periodicity_dir\n# from utils import timer\n\n# module = run_path(str(next(data_dir1.glob(\"*.py\"))))\n# config = module[\"config\"]\n\n# with timer(\"load data\"):\n# test_df = pl.read_parquet(output_dir / \"test_series.parquet\")\n# test_pred_df = pl.read_parquet(output_dir / \"148_gru_scale_factor/test_preds.parquet\")\n\n# assert len(test_df) == len(test_pred_df)\n\n# preds_df = test_df.with_columns(\n# test_pred_df.select([\"prediction_onset\", \"prediction_wakeup\"])\n# ).with_columns(pl.col(\"timestamp\").str.strptime(pl.Datetime, \"%Y-%m-%dT%H:%M:%S%z\"))\n\n# periodicity_dict = {}\n# for series_dir in periodicity_dir.iterdir():\n# series_id = series_dir.stem\n# periodicity = np.load(series_dir / \"periodicity.npy\")\n# periodicity = np.minimum(periodicity, uniform_filter1d(periodicity, size=config.filter_size))\n# periodicity_dict[series_id] = periodicity\n\n# with timer(\"make submission\"):\n# event_dfs = []\n# for series_id, series_df in preds_df.group_by(\"series_id\"):\n# for event in [\"onset\", \"wakeup\"]:\n# event_preds = series_df[f\"prediction_{event}\"].to_numpy().copy()\n# event_preds *= 1 - periodicity_dict[series_id][: len(event_preds)]\n# steps = find_peaks(event_preds, height=config.height, distance=config.distance)[0]\n\n# event_dfs.append(\n# series_df[steps]\n# .with_columns(pl.lit(event).alias(\"event\"))\n# .rename({f\"prediction_{event}\": \"score\"})\n# .select([\"series_id\", \"step\", \"timestamp\", \"event\", \"score\"])\n# )\n\n# submission_df = (\n# pl.concat(event_dfs)\n# .with_columns(pl.col(\"timestamp\").dt.offset_by(\"2h\").dt.date().alias(\"date\"))\n# .with_columns(\n# pl.col(\"score\") / (pl.col(\"score\").sum().over([\"series_id\", \"event\", \"date\"]) + config.daily_score_offset)\n# )\n# .sort([\"series_id\", \"step\"])\n# .with_columns(pl.arange(0, pl.count()).alias(\"row_id\"))\n# .select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n# )\n# submission_df.write_csv(\"submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:07:40.110284Z","iopub.execute_input":"2023-12-05T11:07:40.1124Z","iopub.status.idle":"2023-12-05T11:07:40.121089Z","shell.execute_reply.started":"2023-12-05T11:07:40.112364Z","shell.execute_reply":"2023-12-05T11:07:40.120147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# shimacos","metadata":{}},{"cell_type":"code","source":"%%writefile shimacos_utils.py\nfrom glob import glob\nfrom typing import Any\n\nimport numpy as np\nimport polars as pl\nimport pandas as pd\nfrom numba import jit\nfrom scipy.signal import find_peaks\nfrom tqdm.auto import tqdm\n\ndef make_submission(\n row_idxs: dict[str, Any],\n preds: np.ndarray,\n steps: np.ndarray,\n height: float = 0.001,\n distance: int = 100,\n) -> pl.DataFrame:\n records = []\n for row in tqdm(row_idxs):\n _preds = preds[row[\"row_id\"]]\n _steps = steps[row[\"row_id\"]]\n for i, event_name in enumerate([\"onset\", \"wakeup\"]):\n idxs, peak_heights = find_peaks(_preds[:, i], height=height, distance=distance)\n peak_steps = _steps[idxs]\n if len(peak_steps) > 0:\n records.append(\n {\n \"series_id\": row[\"series_id\"],\n \"event\": event_name,\n \"step\": peak_steps,\n \"score\": peak_heights[\"peak_heights\"],\n }\n )\n else:\n records.append(\n {\n \"series_id\": row[\"series_id\"],\n \"event\": event_name,\n \"step\": _steps[0],\n \"score\": 0.0,\n }\n )\n sub_df = pl.from_pandas(pd.DataFrame(records).explode([\"step\", \"score\"]))\n return sub_df\n\ndef make_chunk_data(sub_df: pl.DataFrame, step_size: int = 360, use_peak_chunk: bool = False) -> pl.DataFrame:\n assert step_size % 12 == 0\n if use_peak_chunk:\n chunk_df = sub_df[[\"series_id\", \"step\"]].unique()\n chunk_df = chunk_df.with_columns(\n pl.col(\"step\").alias(\"peak_step\"),\n pl.col(\"step\").map_elements(lambda x: range(x - step_size, x + step_size, 12)),\n pl.col(\"step\").rank(method=\"ordinal\").over(\"series_id\").alias(\"chunk_id\"),\n )\n chunk_df = chunk_df.explode(\"step\").unique()\n chunk_df = chunk_df.sort([\"series_id\", \"step\"])\n chunk_df = chunk_df.filter(pl.col(\"step\") >= 0)\n chunk_df = chunk_df.with_columns(pl.col(\"step\").cast(pl.Int64))\n else:\n chunk_df = sub_df[[\"series_id\", \"step\"]]\n chunk_df = chunk_df.with_columns(\n pl.col(\"step\").map_elements(lambda x: range(x - step_size, x + step_size, 12))\n )\n chunk_df = chunk_df.explode(\"step\").unique()\n chunk_df = chunk_df.sort([\"series_id\", \"step\"])\n chunk_df = chunk_df.filter(pl.col(\"step\") >= 0)\n chunk_df = chunk_df.with_columns(pl.col(\"step\").cast(pl.Int64))\n chunk_df = chunk_df.with_columns(\n ((pl.col(\"step\") - pl.col(\"step\").shift(1)) != 12)\n .cast(int)\n .cumsum()\n .over(\"series_id\")\n .fill_null(0)\n .alias(\"chunk_id\")\n ) \n\n return chunk_df\n\n\ndef post_process_from_2nd(\n pred_df,\n event_rate: int | float = 500,\n height: float = 0.001,\n event2col: dict[str, str] = {\n \"onset\": \"stacking_prediction_onset\",\n \"wakeup\": \"stacking_prediction_wakeup\",\n },\n event2offset: dict[str, str] = {\"onset\": \"5h\", \"wakeup\": \"0h\"},\n use_daily_norm: bool = True,\n daily_score_offset=10,\n later_date_max_sub_rate: float | None = None,\n):\n \"\"\"\n 1分ごとの予測値を用いてイベントを検出する\n 用語\n - 予測地点: 2段目のモデルによって得られた1分毎の予測位置\n - 候補地点: event の候補となる 15秒 or 45秒始まりの30秒間隔の位置\n\n Args:\n pred_df (pl.DataFrame): timestamp 込み\n event_rate (int | float, optional): [0,1) の値であれば1分間に何回イベントが起こるか。intの場合はseries_idごとに同じイベント数を検出。 Defaults to 0.005.\n height (float, optional): 候補地点の期待値がこの値を下回ったら終了。 Defaults to 0.1.\n event2col (dict[str, str], optional): event名と予測値のカラム名の対応。 Defaults to {\"onset\": \"stacking_prediction_onset\", \"wakeup\": \"stacking_prediction_wakeup\"}.\n weight_rate (float | None, optional): 遠くの予測値の期待値を割り引く際の重み。Noneの場合は重みを1とする。1/weight_rate 倍ずつ遠くの予測値の重みが小さくなっていく。 Defaults to None.\n use_daily_norm (bool, optional): 一日ごとに予測値を正規化するかどうか。 Defaults to False.\n daily_score_offset (float, optional): 正規化の際のoffset。 Defaults to 1.0.\n later_date_max_sub_rate (float | None, optional): 日付が古いほど予測値を割り引く際の最大割引率。Noneの場合は割引しない。 Defaults to None.\n Returns:\n event_df (pl.DataFrame): row_id, series_id, step, event, score をカラムに持つ。\n \"\"\"\n high_match_nums = (0, 1, 3, 5, 8, 10, 13, 15, 20, 25, 30)\n low_match_nums = (0, 1, 3, 5, 7, 10, 12, 15, 20, 25, 30)\n match_sums = np.ones(10)\n result_events_records = []\n\n # event ごとに処理\n for event, event_pred_col in event2col.items():\n \"\"\"\n 元の系列の予測地点(長さN): 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, ..., (N-1)*12\n 15秒から30秒おきのevent候補地点(長さ2N): 3, 9, 15, 21, 27, 33, 39, 45, 51, 57, ..., (N-1)*12+3, (N-1)*12+9\n - 15秒(3step)から1分おき(長さN): 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n - 左の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums\n - 右の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n - 45秒(9step)から1分おき(長さN): 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n - 左の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n - 右の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums\n \"\"\"\n\n # series内でのindexを振り、chunk内での最大と最小を計算\n minute_pred_df = pred_df.filter(pl.col(\"timestamp\").is_not_null())\n\n if use_daily_norm:\n minute_pred_df = (\n minute_pred_df.with_columns(\n pl.col(\"timestamp\").dt.offset_by(event2offset[event]).dt.date().alias(\"date\")\n )\n .with_columns(\n pl.col(event_pred_col).sum().over([\"series_id\", \"date\"]).alias(\"date_sum\")\n )\n .with_columns(\n pl.col(event_pred_col)\n / (pl.col(\"date_sum\") + (1 / (daily_score_offset + pl.col(\"date_sum\"))))\n )\n )\n if later_date_max_sub_rate is not None:\n minute_pred_df = minute_pred_df.with_columns(\n pl.col(\"date\").min().over(\"series_id\").alias(\"min_date\"),\n pl.col(\"date\").max().over(\"series_id\").alias(\"max_date\"),\n ).with_columns(\n pl.col(event_pred_col)\n * (\n pl.lit(1.0)\n - (\n pl.lit(later_date_max_sub_rate)\n * (\n (pl.col(\"date\") - pl.col(\"min_date\")).dt.days().cast(float)\n / (\n (pl.col(\"max_date\") - pl.col(\"min_date\")).dt.days().cast(float)\n + 1.0\n )\n )\n )\n )\n )\n\n max_event_per_series = (\n event_rate if isinstance(event_rate, int) else int(len(minute_pred_df) * event_rate)\n )\n\n # series_id, chunk_id, step でソート\n minute_pred_df = minute_pred_df.sort([\"series_id\", \"chunk_id\", \"step\"])\n\n # 1. 期待値の計算\n # 1.1 左側を計算 (同じindexの予測を含む左側を計算)\n \"\"\"\n 以下をそれぞれ計算する\n - 15秒(3step)から1分おき(長さN)での候補地点での期待値: stepは 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n - 45秒(9step)から1分おき(長さN)での候補地点での期待値: stepは 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n 計算は左側の予測地点の数と、右側の予測地点の数\n \"\"\"\n minute_pred_df = minute_pred_df.with_columns(\n pl.sum_horizontal(\n [\n (\n pl.col(event_pred_col)\n .rolling_sum(window_size=window, center=False, min_periods=1)\n .over([\"series_id\", \"chunk_id\"])\n / match_sums[i]\n )\n for i, window in enumerate(high_match_nums[1:])\n ]\n ).alias(f\"{event}_left_expectation_plus_3step\"),\n pl.sum_horizontal(\n [\n (\n pl.col(event_pred_col)\n .rolling_sum(window_size=window, center=False, min_periods=1)\n .over([\"series_id\", \"chunk_id\"])\n / match_sums[i]\n )\n for i, window in enumerate(low_match_nums[1:])\n ]\n ).alias(f\"{event}_left_expectation_plus_9step\"),\n )\n\n # 1.2 右側を計算(同じindexの予測を含まない右側を計算。逆順にして一個ずらしrolling_sumを取る必要がある）\n minute_pred_df = minute_pred_df.reverse()\n minute_pred_df = minute_pred_df.with_columns(\n pl.sum_horizontal(\n [\n (\n pl.col(event_pred_col)\n .shift(1)\n .rolling_sum(window_size=window, center=False, min_periods=1)\n .over([\"series_id\", \"chunk_id\"])\n .fill_null(0)\n / match_sums[i]\n )\n for i, window in enumerate(low_match_nums[1:])\n ]\n ).alias(f\"{event}_right_expectation_plus_3step\"),\n pl.sum_horizontal(\n [\n (\n pl.col(event_pred_col)\n .shift(1)\n .rolling_sum(window_size=window, center=False, min_periods=1)\n .over([\"series_id\", \"chunk_id\"])\n .fill_null(0)\n / match_sums[i]\n )\n for i, window in enumerate(high_match_nums[1:])\n ]\n ).alias(f\"{event}_right_expectation_plus_9step\"),\n )\n minute_pred_df = minute_pred_df.reverse()\n\n # 合計の期待値計算\n minute_pred_df = minute_pred_df.with_columns(\n (\n pl.col(f\"{event}_left_expectation_plus_3step\")\n + pl.col(f\"{event}_right_expectation_plus_3step\")\n ).alias(f\"{event}_expectation_sum_3step\"),\n (\n pl.col(f\"{event}_left_expectation_plus_9step\")\n + pl.col(f\"{event}_right_expectation_plus_9step\")\n ).alias(f\"{event}_expectation_sum_9step\"),\n )\n\n # print(display(minute_pred_df))\n\n # 3. 最大値の取得 & 期待値の割引\n \"\"\"\n 各予測地点の power を管理する。powerは以下の11種類\n 0: その予測地点が影響を与える範囲は無い\n 1: その予測地点が影響を与える範囲は左右1つ(1min)\n 2: その予測地点が影響を与える範囲は左右3つ\n ︙\n 10: 左右30(step 0~360)\n\n event を作るたびに、eventからtolerance内にある予測地点のpowerを下げる。\n その際に予測地点からtolerance内にある、eventがあったところも含めた候補地点の期待値を割り引く。\n \"\"\"\n for series_id, series_df in tqdm(\n minute_pred_df.select(\n [\n \"series_id\",\n \"chunk_id\",\n \"step\",\n event_pred_col,\n f\"{event}_expectation_sum_3step\",\n f\"{event}_expectation_sum_9step\",\n ]\n ).group_by(\"series_id\"),\n desc=f\"detect {event} peaks\",\n leave=False,\n total=len(minute_pred_df[\"series_id\"].unique()),\n ):\n # chunkごとの id の最大最小を計算\n series_df = series_df.with_row_count().with_columns(\n pl.col(\"row_nr\").max().over([\"chunk_id\"]).alias(\"max_id_in_chunk\"),\n pl.col(\"row_nr\").min().over([\"chunk_id\"]).alias(\"min_id_in_chunk\"),\n )\n\n preds = series_df[event_pred_col].to_numpy()\n expectation_sum_3step = series_df[f\"{event}_expectation_sum_3step\"].to_numpy(\n writable=True\n )\n expectation_sum_9step = series_df[f\"{event}_expectation_sum_9step\"].to_numpy(\n writable=True\n )\n steps = series_df[\"step\"].to_numpy(writable=True)\n step_id_mins = series_df[\"min_id_in_chunk\"].to_numpy(writable=True)\n step_id_maxs = series_df[\"max_id_in_chunk\"].to_numpy(writable=True) + 1\n powers = np.ones(len(expectation_sum_3step), dtype=np.int32) * 10\n\n result_steps, result_scores = detect_events_for_serie(\n height,\n max_event_per_series,\n high_match_nums,\n low_match_nums,\n match_sums,\n steps,\n step_id_mins,\n step_id_maxs,\n preds,\n expectation_sum_3step,\n expectation_sum_9step,\n powers,\n )\n\n for i in range(len(result_steps)):\n result_events_records.append(\n {\n \"series_id\": series_id,\n \"step\": result_steps[i],\n \"event\": event,\n \"score\": result_scores[i],\n }\n )\n\n # print(expectation_sum_3step[max_step_index], expectation_sum_9step[max_step_index])\n\n if len(result_events_records) == 0: # 一つも予測がない場合はdummyを入れる\n result_events_records.append(\n {\n \"series_id\": series_id,\n \"step\": 0,\n \"event\": \"onset\",\n \"score\": 0,\n }\n )\n sub_df = pl.DataFrame(result_events_records).sort(by=[\"series_id\", \"step\"])\n row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n return sub_df\n\n\n@jit(nopython=True, cache=True)\ndef detect_events_for_serie(\n height,\n max_event_per_series,\n high_match_nums,\n low_match_nums,\n match_sums,\n steps,\n step_id_mins,\n step_id_maxs,\n preds,\n expectation_sum_3step,\n expectation_sum_9step,\n powers,\n):\n result_steps = []\n result_scores = []\n for _ in range(max_event_per_series): # 高い順に最大max_event_per_series個のeventを決定\n # 3.1 最大値の取得\n # 合計の期待値が最大のstepを取得\n max_step3 = expectation_sum_3step.argmax()\n max_score3 = expectation_sum_3step[max_step3]\n max_step9 = expectation_sum_9step.argmax()\n max_score9 = expectation_sum_9step[max_step9]\n if max_score3 > max_score9:\n # print('max_score3')\n left_nums = high_match_nums\n right_nums = low_match_nums\n max_step_index = max_step3\n max_score = max_score3\n result_steps.append(steps[max_step_index] + 3)\n result_scores.append(max_score)\n else:\n # print('max_score9')\n left_nums = low_match_nums\n right_nums = high_match_nums\n max_step_index = max_step9\n max_score = max_score9\n result_steps.append(steps[max_step_index] + 9)\n result_scores.append(max_score)\n if max_score < height: # 閾値以下なら終了\n break\n # print(f\"max_step_index:{max_step_index}, max_score:{max_score}\")\n\n # 3.2 期待値の割引\n \"\"\"\n 各予測地点のpowerを修正するとともに、候補地点の期待値を割引く。\n powerが pi まで小さくなることによってその予測値が影響を与える範囲が狭くなる。\n つまり狭くなって範囲から抜けた expectation_sum の値が、その予測値の値*重みの分だけ小さくなる\n \"\"\"\n # 3.2.1 まずはpowerを修正するstepの候補を探す\n target_step_powers = [] # (target_step, pred, base_power, power, step_min, step_max)のリスト\n for pi in range(0, 10):\n # 左側\n for l_diff in range(left_nums[pi], left_nums[pi + 1]):\n target_step_index = max_step_index - l_diff\n if target_step_index < 0:\n break\n pred = preds[target_step_index]\n base_power = powers[target_step_index]\n if base_power > pi: # power が小さくなる場合のみ修正\n target_step_powers.append(\n (\n target_step_index,\n pred,\n base_power,\n pi,\n step_id_mins[target_step_index],\n step_id_maxs[target_step_index],\n )\n )\n # 右側\n for r_diff in range(right_nums[pi] + 1, right_nums[pi + 1] + 1): # 自分自身と同じindexは含めない\n target_step_index = max_step_index + r_diff\n if target_step_index >= len(powers):\n break\n pred = preds[target_step_index]\n base_power = powers[target_step_index]\n if base_power > pi:\n target_step_powers.append(\n (\n target_step_index,\n pred,\n base_power,\n pi,\n step_id_mins[target_step_index],\n step_id_maxs[target_step_index],\n )\n )\n # print('target_step_powers', target_step_powers)\n\n # 3.2.2 対象となる step の power を修正するとともに期待値を割り引く\n \"\"\"\n 予測地点のpowerを下げるとともに、関連する候補地点の期待値を修正する。\n 検出したeventから遠い予測地点の場合は、予測地点に近い候補地点であっても期待値はその分割り引かれる。\n 3stepの修正をする時は target_stepから左側が low_match_nums, 右側が high_match_nums\n 9stepの修正をする時は target_stepから左側が high_match_nums, 右側が low_match_nums\n だんだんと内側のみがのこるように修正する。\n\n - powerが 10 → 8 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~20個に影響を及ぼすようになる。また、powerが2個減った分全体の期待値も割り引かれる\n - powerが 10 → 5 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~12(13)個に影響を及ぼすようになる\n - powerが 8 → 7 になるケースは左右1~20個に影響を及ぼしていたものが、左右の1~15個に影響を及ぼすようになる\n \"\"\"\n for si, pred, base_power, power, step_min, step_max in target_step_powers:\n # print(f\"si:{si}, pred:{pred}, base_power:{base_power}, power:{power}\")\n powers[si] = power\n # 中心ほど重みが強いので power ごとに処理\n for pi in range(base_power, power, -1): # base_powerからpowerに減らしていくことで予測値の外側から削る\n # 3step\n left_diff_max = low_match_nums[pi]\n right_diff_max = high_match_nums[pi]\n expectation_sum_3step[\n max(si - left_diff_max, step_min) : min(si + right_diff_max, step_max)\n ] -= (pred / match_sums[pi - 1])\n \"\"\"\n if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n print(f'3step pi:{pi}')\n print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\")\n print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n print()\n \"\"\"\n # 9step\n left_diff_max = high_match_nums[pi]\n right_diff_max = low_match_nums[pi]\n expectation_sum_9step[\n max(si - left_diff_max, step_min) : min(si + right_diff_max, step_max)\n ] -= (pred / match_sums[pi - 1])\n \"\"\"\n if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n print(f'9step pi:{pi}')\n print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\")\n print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n print()\n \"\"\"\n return result_steps, result_scores","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:09:04.747584Z","iopub.execute_input":"2023-12-05T11:09:04.748221Z","iopub.status.idle":"2023-12-05T11:09:04.768937Z","shell.execute_reply.started":"2023-12-05T11:09:04.748188Z","shell.execute_reply":"2023-12-05T11:09:04.767935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile feature_creator.py\n\nfrom itertools import chain\n\nimport polars as pl\n\n\nclass FeatureCreator:\n def __init__(self, signal_names: list[str], timeseries_cols: list[str]) -> None:\n self.signal_names = signal_names\n self.timeseries_cols = timeseries_cols\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n return df\n\n\nclass LIDFeatureCreator(FeatureCreator):\n def __init__(\n self, signal_names: list[str], timeseries_cols: list[str], window_size: int = 120\n ) -> None:\n self.signal_names = signal_names\n self.timeseries_cols = timeseries_cols\n self.window_size = window_size\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n # sum over 10minutes\n df = df.with_columns(\n pl.max_horizontal(pl.col(\"enmo\") - 0.02, 0)\n .rolling_sum(self.window_size)\n .over(self.timeseries_cols)\n .alias(\"activity_count\")\n .backward_fill()\n ).with_columns((100 / (pl.col(\"activity_count\") + 1)).alias(\"lid\"))\n return df\n\n\nclass TimeSeriesStatsFeatureCreator:\n def __init__(\n self, signal_names: list[str], window_sizes: list[int], timeseries_cols: list[str]\n ):\n self.signal_names = signal_names\n self.window_sizes = window_sizes\n self.timeseries_cols = timeseries_cols\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n agg_cols = list(\n chain.from_iterable(\n [\n [\n pl.col(column)\n .rolling_mean(window_size, center=True, min_periods=1)\n .over(self.timeseries_cols)\n .suffix(f\"_{window_size}_rolling_mean\"),\n pl.col(column)\n .rolling_std(window_size, center=True, min_periods=1)\n .over(self.timeseries_cols)\n .suffix(f\"_{window_size}_rolling_std\"),\n pl.col(column)\n .rolling_max(window_size, center=True, min_periods=1)\n .over(self.timeseries_cols)\n .suffix(f\"_{window_size}_rolling_max\"),\n ]\n for window_size in self.window_sizes\n for column in self.signal_names\n ]\n )\n )\n df = df.with_columns(agg_cols)\n return df\n\n\nclass TimeSeriesStatsDiffFeatureCreator:\n def __init__(\n self, signal_names: list[str], window_sizes: list[int], timeseries_cols: list[str]\n ):\n self.signal_names = signal_names\n self.window_sizes = window_sizes\n self.timeseries_cols = timeseries_cols\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n agg_cols = list(\n chain.from_iterable(\n [\n [\n (\n pl.col(column)\n .rolling_mean(window_size, min_periods=1)\n .over(self.timeseries_cols)\n - pl.col(column)\n ).suffix(f\"_diff_prev{window_size}_rolling_mean\"),\n ]\n for window_size in self.window_sizes\n for column in self.signal_names\n ]\n )\n )\n df = df.with_columns(agg_cols)\n agg_cols = list(\n chain.from_iterable(\n [\n [\n (\n pl.col(column)\n .rolling_mean(window_size, closed=\"right\", min_periods=1)\n .over(self.timeseries_cols)\n - pl.col(column)\n ).suffix(f\"_diff_lead{window_size}_rolling_mean\"),\n ]\n for window_size in self.window_sizes\n for column in self.signal_names\n ]\n )\n )\n df = df.with_columns(agg_cols)\n return df\n\n\nclass TimeSeriesDiffFeatureCreator:\n def __init__(\n self, signal_names: list[str], window_sizes: list[int], timeseries_cols: list[str]\n ):\n self.signal_names = signal_names\n self.window_sizes = window_sizes\n self.timeseries_cols = timeseries_cols\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n df = df.with_columns(\n [\n (pl.col(column) - pl.col(column).shift(1).over(self.timeseries_cols))\n .backward_fill()\n .suffix(\"_diff_lag\")\n for column in self.signal_names\n ]\n )\n return df\n\n\nclass TimeFeatureCreator:\n def __init__(\n self, signal_names: list[str], window_sizes: list[int], timeseries_cols: list[str]\n ):\n self.signal_names = signal_names\n self.window_sizes = window_sizes\n self.timeseries_cols = timeseries_cols\n\n def create(self, df: pl.DataFrame) -> pl.DataFrame:\n df = df.with_columns(\n (pl.col(\"timestamp\").dt.hour() + pl.col(\"timestamp\").dt.minute()).alias(\"time_idx\"),\n pl.col(\"timestamp\").dt.hour().alias(\"hour\"),\n pl.col(\"timestamp\").dt.minute().alias(\"minute\"),\n (pl.col(\"timestamp\").dt.minute() % 15).alias(\"minute_mod15\"),\n pl.col(\"timestamp\").dt.weekday().alias(\"weekday\"),\n )\n return df\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:09:04.853362Z","iopub.execute_input":"2023-12-05T11:09:04.853671Z","iopub.status.idle":"2023-12-05T11:09:04.862174Z","shell.execute_reply.started":"2023-12-05T11:09:04.853645Z","shell.execute_reply":"2023-12-05T11:09:04.861273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile preprocess.py\n\nimport os\nimport pickle\nimport ctypes\nimport gc\nimport yaml\n\nimport polars as pl\nimport numpy as np\nfrom settings import output_dir, shimacos_output_dir\nfrom utils import timer\nfrom shimacos_utils import make_submission, make_chunk_data\nfrom feature_creator import (\n LIDFeatureCreator,\n TimeFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesStatsFeatureCreator,\n)\n\nclass Preprocessor:\n def __init__(\n self, sakami_model_names: list[str], kami_model_names: list[str], downsample: bool\n ) -> None:\n self.sakami_model_names = sakami_model_names\n self.kami_model_names = kami_model_names\n self.model_names = sakami_model_names + kami_model_names\n self.downsample = downsample\n self.series_df = pl.read_parquet(output_dir / \"test_series.parquet\").with_columns(\n pl.col(\"timestamp\").str.to_datetime(),\n pl.Series('periodicity', np.load(output_dir / \"periodicity.npy\") - 1),\n )\n if downsample:\n self.series_df = self.series_df.with_columns(\n pl.col(\"timestamp\").alias(\"raw_timestamp\"), pl.col(\"timestamp\").dt.truncate(\"1m\")\n )\n\n def _load_sakami_data(self) -> pl.DataFrame:\n dfs = []\n for exp_name in self.sakami_model_names:\n df = (\n pl.read_parquet(f\"/kaggle/temp/sakami-output/{exp_name}/test_preds.parquet\")\n .sort(\"row_id\")\n .drop(\"row_id\")\n )\n has_sleep = \"sleep\" in df.columns\n if has_sleep:\n df = df.rename(\n {\n \"prediction_wakeup\": f\"pred_wakeup_{exp_name}\",\n \"prediction_onset\": f\"pred_onset_{exp_name}\",\n \"sleep\": f\"pred_sleep_{exp_name}\",\n }\n )\n else:\n df = df.rename(\n {\n \"prediction_wakeup\": f\"pred_wakeup_{exp_name}\",\n \"prediction_onset\": f\"pred_onset_{exp_name}\",\n }\n )\n if self.downsample:\n _df = pl.concat([df, self.series_df[[\"series_id\", \"raw_timestamp\", \"timestamp\"]]], how=\"horizontal\")\n if not has_sleep:\n pred_cols = [f\"pred_onset_{exp_name}\", f\"pred_wakeup_{exp_name}\"]\n else:\n pred_cols = [f\"pred_onset_{exp_name}\", f\"pred_wakeup_{exp_name}\", f\"pred_sleep_{exp_name}\"] \n df = _df.group_by([\"series_id\", \"timestamp\"], maintain_order=True).agg(pl.col(pred_cols).mean(),)\n _df = (\n _df.drop(\"timestamp\")\n .rename({\"raw_timestamp\": \"timestamp\"})\n .select(\n pl.col([\"series_id\", \"timestamp\"]),\n pl.col(pred_cols).suffix(\"_raw\"),\n )\n )\n df = df.join(_df, on=[\"series_id\", \"timestamp\"], how=\"left\")\n df = df.drop(['series_id', 'timestamp'])\n dfs.append(df)\n df = pl.concat(dfs, how=\"horizontal\")\n return df\n\n def _load_kami_data(self) -> pl.DataFrame:\n # kamiモデルはtrain, testで処理分岐してる\n# mode = os.getenv('MODE', 'test')\n mode = 'test'\n dfs = []\n for exp_name in self.kami_model_names:\n df = pl.read_parquet(\n f\"/kaggle/temp/kami_output/cv_inference/{exp_name}/single/{mode}_pred.parquet\"\n ).rename(\n {\n \"pred_sleep\": f\"pred_sleep_{exp_name}\",\n \"pred_wakeup\": f\"pred_wakeup_{exp_name}\",\n \"pred_onset\": f\"pred_onset_{exp_name}\",\n }\n )\n if self.downsample:\n _df = pl.concat([df, self.series_df[[\"series_id\", \"raw_timestamp\", \"timestamp\"]]], how=\"horizontal\")\n df = _df.group_by([\"series_id\", \"timestamp\"], maintain_order=True).agg(\n pl.col(f\"pred_onset_{exp_name}\").mean(),\n pl.col(f\"pred_wakeup_{exp_name}\").mean(),\n pl.col(f\"pred_sleep_{exp_name}\").mean(),\n )\n _df = (\n _df.drop(\"timestamp\")\n .rename({\"raw_timestamp\": \"timestamp\"})\n .select(\n pl.col([\"series_id\", \"timestamp\"]),\n pl.col([f\"pred_onset_{exp_name}\", f\"pred_wakeup_{exp_name}\", f\"pred_sleep_{exp_name}\"]).suffix(\"_raw\"),\n )\n )\n df = df.join(_df, on=[\"series_id\", \"timestamp\"], how=\"left\")\n df = df.drop(['series_id', 'timestamp'])\n dfs.append(df)\n df = pl.concat(dfs, how=\"horizontal\")\n return df\n\n def load_data(self):\n with timer(\"load sakami data\"):\n sakami_df = self._load_sakami_data()\n with timer(\"load kami data\"):\n kami_df = self._load_kami_data()\n if self.downsample:\n series_df = self.series_df.group_by(\n [\"series_id\", \"timestamp\"], maintain_order=True\n ).agg(\n pl.col([\"anglez\", \"enmo\"]).mean(),\n pl.col(\"step\").first(),\n pl.col(\"periodicity\").first(),\n )\n else:\n series_df = self.series_df\n pred_df = pl.concat([sakami_df, kami_df, series_df], how=\"horizontal\")\n return pred_df\n \n def create_chunk_data(self, pred_df: pl.DataFrame, step_size: int, use_peak_chunk: bool) -> pl.DataFrame: \n _df = pred_df.filter(pl.col(\"periodicity\") == 0)\n _df = _df.with_columns(pl.Series(\"row_id\", range(len(_df))))\n\n row_idxs = _df.group_by([\"series_id\"]).agg(pl.col([\"row_id\"])).rows(named=True)\n preds = _df[[\"prediction_onset\", \"prediction_wakeup\"]].to_numpy()\n steps = _df[\"step\"].to_numpy()\n sub_df = make_submission(row_idxs, preds, steps, distance=8)\n sub_df = sub_df.filter(pl.col(\"step\") != 0)\n chunk_df = make_chunk_data(sub_df, step_size=step_size, use_peak_chunk=use_peak_chunk)\n return chunk_df\n\n\n def create_feature(self, pred_df: pl.DataFrame, step_size: int = 360, use_peak_chunk: bool = False) -> pl.DataFrame:\n with timer(\"create pred feature\"):\n pred_df = pred_df.with_columns(\n (\n pl.sum_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_onset\"),\n (\n pl.min_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_onset_min\"),\n (\n pl.max_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_onset_max\"),\n (\n pl.sum_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_wakeup\"),\n (\n pl.min_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_wakeup_min\"),\n (\n pl.max_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_wakeup_max\"),\n )\n chunk_df = self.create_chunk_data(pred_df, step_size, use_peak_chunk)\n pred_df = pred_df.with_columns(pl.col('step').cast(pl.Int64))\n chunk_df = chunk_df.join(pred_df, on=[\"series_id\", \"step\"], how=\"left\")\n chunk_df = chunk_df.filter(pl.col('timestamp').is_not_null())\n\n # ===============\n # 特徴量生成\n # ===============\n with timer(\"create lid\"):\n creator = LIDFeatureCreator(\n [], timeseries_cols=[\"series_id\", \"chunk_id\"], window_size=5\n )\n chunk_df = creator.create(chunk_df)\n\n with timer(\"create timeseries features\"):\n for creator in [\n TimeSeriesStatsFeatureCreator,\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n TimeFeatureCreator,\n ]:\n chunk_df = creator(\n signal_names=[\n \"enmo\",\n \"anglez\",\n \"lid\",\n ],\n window_sizes=[3],\n timeseries_cols=[\"series_id\", \"chunk_id\"],\n ).create(chunk_df)\n with timer(\"create timeseries features (pred)\"):\n for creator in [\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n ]:\n chunk_df = creator(\n signal_names=[f\"pred_onset_{model_name}\" for model_name in self.model_names]\n + [f\"pred_wakeup_{model_name}\" for model_name in self.model_names],\n window_sizes=[3, 5],\n timeseries_cols=[\"series_id\", \"chunk_id\"],\n ).create(chunk_df)\n return chunk_df\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:32:15.824875Z","iopub.execute_input":"2023-12-05T11:32:15.825244Z","iopub.status.idle":"2023-12-05T11:32:15.837326Z","shell.execute_reply.started":"2023-12-05T11:32:15.825214Z","shell.execute_reply":"2023-12-05T11:32:15.836351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 009のデータセット作成","metadata":{}},{"cell_type":"code","source":"# %%python\n\n# import os\n# import pickle\n# import ctypes\n# import gc\n# import yaml\n\n# import polars as pl\n# import numpy as np\n# from settings import output_dir, shimacos_output_dir, sakami_model_names_009, kami_model_names_009\n# from utils import timer\n# from shimacos_utils import make_submission, make_chunk_data\n# from feature_creator import (\n# LIDFeatureCreator,\n# TimeFeatureCreator,\n# TimeSeriesDiffFeatureCreator,\n# TimeSeriesStatsDiffFeatureCreator,\n# TimeSeriesStatsFeatureCreator,\n# )\n# from preprocess import Preprocessor\n\n# if __name__ == '__main__':\n# preprocessor = Preprocessor(\n# sakami_model_names_009,\n# kami_model_names_009,\n# downsample=True\n# )\n# pred_df = preprocessor.load_data()\n# chunk_df = preprocessor.create_feature(pred_df, step_size=96, use_peak_chunk=False)\n# with open(shimacos_output_dir / '009_stacking_data.pkl', \"wb\") as f:\n# pickle.dump(chunk_df, f)\n# del chunk_df; gc.collect()\n# libc = ctypes.CDLL(\"libc.so.6\")\n# libc.malloc_trim(0)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:32:17.187179Z","iopub.execute_input":"2023-12-05T11:32:17.187885Z","iopub.status.idle":"2023-12-05T11:32:30.985677Z","shell.execute_reply.started":"2023-12-05T11:32:17.187851Z","shell.execute_reply":"2023-12-05T11:32:30.984853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 030のデータセット作成","metadata":{}},{"cell_type":"code","source":"%%python\n\nimport os\nimport pickle\nimport ctypes\nimport gc\nimport yaml\n\nimport polars as pl\nimport numpy as np\nfrom settings import output_dir, shimacos_output_dir, sakami_model_names_030, kami_model_names_030\nfrom utils import timer\nfrom shimacos_utils import make_submission, make_chunk_data\nfrom feature_creator import (\n LIDFeatureCreator,\n TimeFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesStatsFeatureCreator,\n)\nfrom preprocess import Preprocessor\n\nclass PreprocessorV2(Preprocessor):\n def __init__(\n self, sakami_model_names: list[str], kami_model_names: list[str], downsample: bool\n ) -> None:\n self.sakami_model_names = sakami_model_names\n self.kami_model_names = kami_model_names\n self.model_names = sakami_model_names + kami_model_names\n self.downsample = downsample\n self.series_df = pl.read_parquet(output_dir / \"test_series.parquet\").with_columns(\n pl.col(\"timestamp\").str.to_datetime(),\n pl.Series('periodicity', np.load(output_dir / \"periodicity.npy\") - 1),\n )\n if downsample:\n self.series_df = self.series_df.with_columns(\n pl.col(\"timestamp\").alias(\"raw_timestamp\"), pl.col(\"timestamp\").dt.truncate(\"1m\")\n )\n \n def create_feature(self, pred_df: pl.DataFrame, step_size: int = 360, use_peak_chunk: bool = False) -> pl.DataFrame:\n with timer(\"create pred feature\"):\n pred_df = pred_df.with_columns(\n (\n pl.sum_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_onset\"),\n (\n pl.min_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n ).alias(\"prediction_onset_min\"),\n (\n pl.max_horizontal([f\"pred_onset_{model_name}\" for model_name in self.model_names])\n ).alias(\"prediction_onset_max\"),\n (\n pl.sum_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n / len(self.model_names)\n ).alias(\"prediction_wakeup\"),\n (\n pl.min_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n ).alias(\"prediction_wakeup_min\"),\n (\n pl.max_horizontal([f\"pred_wakeup_{model_name}\" for model_name in self.model_names])\n ).alias(\"prediction_wakeup_max\"),\n ).with_columns(\n (\n pl.sum_horizontal(\n [\n (pl.col(\"prediction_onset\") - pl.col(f\"pred_onset_{model_name}\")) ** 2\n for model_name in self.model_names\n ]\n )\n / len(self.model_names)\n ).alias(\"prediction_onset_var\"),\n (\n pl.sum_horizontal(\n [\n (pl.col(\"prediction_wakeup\") - pl.col(f\"pred_wakeup_{model_name}\")) ** 2\n for model_name in self.model_names\n ]\n )\n / len(self.model_names)\n ).alias(\"prediction_wakeup_var\"),\n )\n chunk_df = self.create_chunk_data(pred_df, step_size, use_peak_chunk)\n pred_df = pred_df.with_columns(pl.col('step').cast(pl.Int64))\n chunk_df = chunk_df.join(pred_df, on=[\"series_id\", \"step\"], how=\"left\")\n chunk_df = chunk_df.filter(pl.col('timestamp').is_not_null())\n\n # ===============\n # 特徴量生成\n # ===============\n with timer(\"create lid\"):\n creator = LIDFeatureCreator(\n [], timeseries_cols=[\"series_id\", \"chunk_id\"], window_size=5\n )\n chunk_df = creator.create(chunk_df)\n\n with timer(\"create timeseries features\"):\n for creator in [\n TimeSeriesStatsFeatureCreator,\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n TimeFeatureCreator,\n ]:\n chunk_df = creator(\n signal_names=[\n \"enmo\",\n \"anglez\",\n \"lid\",\n ],\n window_sizes=[3],\n timeseries_cols=[\"series_id\", \"chunk_id\"],\n ).create(chunk_df)\n with timer(\"create timeseries features (pred)\"):\n for creator in [\n TimeSeriesStatsDiffFeatureCreator,\n TimeSeriesDiffFeatureCreator,\n ]:\n chunk_df = creator(\n signal_names=[f\"pred_onset_{model_name}\" for model_name in self.model_names]\n + [f\"pred_wakeup_{model_name}\" for model_name in self.model_names],\n window_sizes=[3, 5],\n timeseries_cols=[\"series_id\", \"chunk_id\"],\n ).create(chunk_df)\n return chunk_df\n\nif __name__ == '__main__':\n preprocessor = PreprocessorV2(\n sakami_model_names_030,\n kami_model_names_030,\n downsample=True\n )\n pred_df = preprocessor.load_data()\n chunk_df = preprocessor.create_feature(pred_df, step_size=96, use_peak_chunk=False)\n with open(shimacos_output_dir / '030_stacking_data.pkl', \"wb\") as f:\n pickle.dump(chunk_df, f)\n del chunk_df; gc.collect()\n libc = ctypes.CDLL(\"libc.so.6\")\n libc.malloc_trim(0)","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:38:27.907916Z","iopub.execute_input":"2023-12-05T11:38:27.908721Z","iopub.status.idle":"2023-12-05T11:38:44.799872Z","shell.execute_reply.started":"2023-12-05T11:38:27.908688Z","shell.execute_reply":"2023-12-05T11:38:44.799031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls /kaggle/temp/shimacos-output","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:39:00.278502Z","iopub.execute_input":"2023-12-05T11:39:00.279178Z","iopub.status.idle":"2023-12-05T11:39:01.223402Z","shell.execute_reply.started":"2023-12-05T11:39:00.279147Z","shell.execute_reply":"2023-12-05T11:39:01.22226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q --no-deps /kaggle/input/shimacos-pip-wheels/lightgbm-4.0.0-py3-none-manylinux_2_28_x86_64.whl ","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:39:02.514173Z","iopub.execute_input":"2023-12-05T11:39:02.515157Z","iopub.status.idle":"2023-12-05T11:39:24.788236Z","shell.execute_reply.started":"2023-12-05T11:39:02.515114Z","shell.execute_reply":"2023-12-05T11:39:24.786915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport pandas as pd\nimport polars as pl\nimport numpy as np\nimport lightgbm as lgb\nfrom settings import shimacos_output_dir\nfrom shimacos_utils import post_process_from_2nd\nfrom omegaconf import OmegaConf\n\nif __name__ == '__main__':\n nb_name = 'shimacos-stacking-exp013-009-1minute-sampling'\n model_name = 'stacking_exp059_030_truncate_lgbm'\n config = OmegaConf.load(f'/kaggle/input/{nb_name}/models/{model_name}/config.yaml')\n df: pl.DataFrame = pd.read_pickle(shimacos_output_dir / '030_stacking_data.pkl')\n for event in ['wakeup', 'onset']:\n bst: lgb.Booster = pd.read_pickle(f'/kaggle/input/{nb_name}/models/{model_name}/booster_{event}.pkl')\n preds = np.zeros(len(df))\n for i in range(5):\n preds += bst[i].predict(df[bst[0].feature_name()].to_numpy()) / 5\n df = df.with_columns(pl.Series(f'stacking_prediction_{event}', preds))\n df[\n ['series_id', 'timestamp', 'step', 'chunk_id', 'stacking_prediction_onset', 'stacking_prediction_wakeup']\n ].write_parquet(shimacos_output_dir / f'{model_name}_result.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:40:42.319509Z","iopub.execute_input":"2023-12-05T11:40:42.319871Z","iopub.status.idle":"2023-12-05T11:41:01.071444Z","shell.execute_reply.started":"2023-12-05T11:40:42.319843Z","shell.execute_reply":"2023-12-05T11:41:01.070657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\nimport pandas as pd\nimport polars as pl\nimport numpy as np\nimport catboost as cat\nfrom settings import shimacos_output_dir\nfrom shimacos_utils import post_process_from_2nd\nfrom omegaconf import OmegaConf\n\nif __name__ == '__main__':\n nb_name = 'shimacos-stacking-exp013-009-1minute-sampling'\n model_name = 'stacking_exp061_030_truncate_cat'\n config = OmegaConf.load(f'/kaggle/input/{nb_name}/models/{model_name}/config.yaml')\n feature_cols = config.feature.numerical_cols + config.feature.cat_cols\n df: pl.DataFrame = pd.read_pickle(shimacos_output_dir / '030_stacking_data.pkl')\n for event in ['wakeup', 'onset']:\n bst: cat.CatBoost = pd.read_pickle(f'/kaggle/input/{nb_name}/models/{model_name}/booster_label_{event}.pkl')\n preds = np.zeros(len(df))\n data = cat.Pool(\n df[feature_cols].to_pandas(),\n feature_names=feature_cols,\n cat_features=config.feature.cat_cols,\n )\n for i in range(5):\n preds += bst[i].predict(data, prediction_type=\"Probability\")[:, 1] / 5\n df = df.with_columns(pl.Series(f'stacking_prediction_{event}', preds))\n df[\n ['series_id', 'timestamp', 'step', 'chunk_id', 'stacking_prediction_onset', 'stacking_prediction_wakeup']\n ].write_parquet(shimacos_output_dir / f'{model_name}_result.parquet')\n# sub_df = post_process_from_2nd(df)\n# sub_df.write_csv('cat_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:43:25.767545Z","iopub.execute_input":"2023-12-05T11:43:25.767941Z","iopub.status.idle":"2023-12-05T11:43:32.002818Z","shell.execute_reply.started":"2023-12-05T11:43:25.767908Z","shell.execute_reply":"2023-12-05T11:43:32.00181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%python \n# from typing import Any\n\n# import pandas as pd\n# import polars as pl\n# import numpy as np\n# import catboost as cat\n# import torch\n# import torch.nn as nn\n# from tqdm.auto import trange\n# from omegaconf import OmegaConf, DictConfig\n# from torch.utils.data.dataloader import DataLoader\n# import torch.nn.functional as F\n\n# from settings import shimacos_output_dir\n# from shimacos_utils import post_process_from_2nd\n\n\n# class StackingDataset(object):\n# def __init__(self, config: DictConfig, mode: str = \"test\"):\n# self.config = config\n# self.mode = mode\n# self.refinement_step = False\n# self.le_dict = {}\n\n# # Data load\n# df: pl.DataFrame = pd.read_pickle(shimacos_output_dir / \"030_stacking_data.pkl\")\n# df = df.filter(pl.col(\"timestamp\").is_not_null())\n# df = df.with_columns(pl.Series(\"row_idx\", np.arange(len(df))))\n# features = df[config.feature.numerical_cols + config.feature.cat_cols].to_numpy()\n\n# self.id_col = 'series_id'\n\n# self.numerical_features = features[:, : len(config.feature.numerical_cols)]\n# self.cat_features = features[:, len(config.feature.numerical_cols) :]\n\n# self.series_step_timestamps = df[[\"series_id\", \"step\", \"timestamp\"]].unique()\n\n# row_idxs_list = (\n# df.group_by([\"series_id\", \"chunk_id\"], maintain_order=True)\n# .agg(pl.col(\"row_idx\"))[\"row_idx\"]\n# .to_numpy()\n# )\n# self.ids = df[self.id_col].to_numpy().flatten()\n\n# self.steps = df[\"step\"].to_numpy()\n# self.numerial_cols = config.feature.numerical_cols\n# self.window_size = config.window_size\n# self.chunk_size = config.chunk_size\n\n# self.indices = self._create_indices(row_idxs_list, False)\n\n# def _create_indices(\n# self,\n# row_idxs_list,\n# use_only_positive_chunk: bool = False,\n# ):\n# indices = []\n# for row_idxs in row_idxs_list:\n# if len(row_idxs) > self.chunk_size:\n# for i in range(0, len(row_idxs), self.window_size):\n# idxs = row_idxs[i : i + self.chunk_size]\n# if use_only_positive_chunk and self.mode == \"train\":\n# if np.sum(self.labels[idxs]) > 0:\n# indices.append(idxs)\n# else:\n# indices.append(idxs)\n# else:\n# if use_only_positive_chunk and self.mode == \"train\":\n# if np.sum(self.labels[row_idxs]) > 0:\n# indices.append(row_idxs)\n# else:\n# indices.append(row_idxs)\n\n# return indices\n\n\n# def __len__(self):\n# return len(self.indices)\n\n# def __getitem__(self, idx: int) -> dict[str, Any]:\n# indices = self.indices[idx]\n# ids = self.ids[indices].tolist()\n# steps = self.steps[indices]\n# numerical_features = self.numerical_features[indices]\n# cat_features = self.cat_features[indices]\n# seq_len = len(numerical_features)\n# attention_mask = np.ones(seq_len)\n# if seq_len < self.chunk_size:\n# numerical_features = np.pad(\n# numerical_features, ((0, self.chunk_size - seq_len), (0, 0))\n# )\n# cat_features = np.pad(cat_features, ((0, self.chunk_size - seq_len), (0, 0)))\n# steps = np.pad(steps, (0, self.chunk_size - seq_len), constant_values=-1)\n# attention_mask = np.pad(\n# attention_mask, (0, self.chunk_size - seq_len), constant_values=0\n# )\n\n# out = {\n# self.id_col: ids[0],\n# \"numerical_feature\": numerical_features.astype(np.float32),\n# \"cat_feature\": cat_features.astype(np.float32),\n# \"step\": steps.astype(np.float32),\n# \"attention_mask\": attention_mask,\n# }\n# return out\n\n# class SEScale(nn.Module):\n# def __init__(self, ch: int, r: int) -> None:\n# super().__init__()\n# self.fc1 = nn.Linear(ch, r)\n# self.fc2 = nn.Linear(r, ch)\n\n# def forward(self, x: torch.FloatTensor) -> torch.FloatTensor:\n# h = self.fc1(x)\n# h = F.relu(h)\n# h = self.fc2(h).sigmoid()\n# return h * x\n\n \n# class ResidualGRU(nn.Module):\n# def __init__(self, hidden_size: int, n_layers: int = 1, bidirectional: bool = True) -> None:\n# super().__init__()\n# self.hidden_size = hidden_size\n# self.n_layers = n_layers\n\n# self.gru = nn.GRU(\n# hidden_size,\n# hidden_size,\n# n_layers,\n# batch_first=True,\n# bidirectional=bidirectional,\n# )\n# dir_factor = 2 if bidirectional else 1\n# self.fc = nn.Sequential(\n# nn.Linear(hidden_size * dir_factor, hidden_size),\n# nn.LayerNorm(hidden_size),\n# nn.PReLU(),\n# )\n\n# def forward(self, x, h=None):\n# res, _ = self.gru(x, h)\n# res = self.fc(res)\n# res = res + x\n# return res, _\n\n# class ResNet(nn.Module):\n# def __init__(self, in_features, out_features, in_length, kernel_size, dropout=0.0):\n# super(ResNet, self).__init__()\n# assert kernel_size % 2 == 1\n# self.conv1 = nn.Sequential(\n# nn.Conv1d(\n# in_features,\n# out_features,\n# kernel_size,\n# stride=1,\n# padding=(kernel_size - 1) // 2,\n# ),\n# nn.LayerNorm([out_features, in_length]),\n# nn.ReLU(),\n# nn.Dropout(dropout),\n# )\n# self.conv2 = nn.Sequential(\n# nn.Conv1d(\n# out_features,\n# out_features,\n# kernel_size,\n# stride=1,\n# padding=(kernel_size - 1) // 2,\n# ),\n# nn.LayerNorm([out_features, in_length]),\n# nn.ReLU(),\n# nn.Dropout(dropout),\n# )\n\n# def forward(self, x):\n# x = self.conv1(x)\n# out = self.conv2(x) + x\n# return out\n\n# class CNNRNNWoDownSample(nn.Module):\n# def __init__(self, config):\n# super(CNNRNNWoDownSample, self).__init__()\n# self.config = config\n# self.category_embeddings = nn.ModuleList(\n# [nn.Embedding(1440, 32, padding_idx=0) for _ in range(config.num_category)]\n# )\n# input_size = 32 * config.num_category + config.num_feature\n\n# self.input_linear = nn.Sequential(\n# SEScale(input_size, 64),\n# nn.Linear(input_size, config.hidden_size),\n# nn.Dropout(config.dropout_rate),\n# nn.ReLU(),\n# )\n\n# self.conv = nn.Sequential(\n# *[\n# ResNet(\n# config.hidden_size,\n# config.hidden_size,\n# in_length=config.max_seq_len,\n# kernel_size=kernel_size,\n# dropout=config.dropout_rate,\n# )\n# for kernel_size in config.kernel_sizes\n# ],\n# )\n# self.gru_layers = nn.ModuleList(\n# [\n# ResidualGRU(config.hidden_size, n_layers=1, bidirectional=True)\n# for _ in range(config.num_layers)\n# ]\n# )\n\n# self.output_linear = nn.Sequential(\n# nn.Dropout(config.dropout_rate),\n# nn.ReLU(),\n# nn.Linear(config.hidden_size, 32),\n# nn.Dropout(config.dropout_rate),\n# nn.ReLU(),\n# nn.Linear(32, config.num_label),\n# )\n\n# def forward(self, batch: dict[str, torch.Tensor]) -> torch.Tensor:\n# # [bs, seq_len, num_feature]\n# numerical_feature = batch[\"numerical_feature\"]\n# cat_features = [\n# embedding(batch[\"cat_feature\"][:, :, i].long())\n# for i, embedding in enumerate(self.category_embeddings)\n# ]\n# out = self.input_linear(torch.cat([numerical_feature] + cat_features, dim=-1))\n# out = self.conv(out.transpose(2, 1)).transpose(2, 1)\n# for gru in self.gru_layers:\n# out, _ = gru(out)\n# out = self.output_linear(out).squeeze(-1)\n\n# return out\n\n\n# def sigmoid(a):\n# return 1 / (1 + np.exp(-a))\n\n# if __name__ == '__main__':\n# nb_name = 'shimacos-stacking-exp013-009-1minute-sampling'\n# model_name = 'stacking_exp060_030_truncate_small'\n# config = OmegaConf.load(f'/kaggle/input/{nb_name}/models/{model_name}/fold0/config.yaml')\n# dataset = StackingDataset(config.data, mode='test')\n# dataloader = DataLoader(dataset, batch_size=32, shuffle=False)\n# config.model.num_feature = len(config.feature.numerical_cols)\n# config.model.num_category = len(config.feature.cat_cols)\n# model = CNNRNNWoDownSample(config.model).cuda()\n# model.eval()\n# dfs = []\n# for fold in trange(5):\n# ckpt = torch.load(f'/kaggle/input/{nb_name}/models/{model_name}/fold{fold}/nn.ckpt')\n# state_dict = {key.replace('model.', ''): val for key, val in ckpt['state_dict'].items() if 'loss' not in key}\n# model.load_state_dict(state_dict, strict=True)\n# ids = []\n# steps = []\n# preds = []\n# for batch in dataloader:\n# batch['numerical_feature'] = batch['numerical_feature'].cuda()\n# batch['cat_feature'] = batch['cat_feature'].cuda()\n# with torch.inference_mode():\n# pred = model(batch)\n# ids.append(batch['series_id'])\n# steps.append(batch['step'].detach().cpu().numpy())\n# preds.append(pred.detach().cpu().numpy())\n# ids = np.concatenate(ids)\n# steps = np.concatenate(steps)\n# preds = np.concatenate(preds)\n# df = pl.DataFrame({'series_id': ids, 'step': steps, 'stacking_prediction_onset': preds[..., 0], 'stacking_prediction_wakeup': preds[..., 1]})\n# df = df.explode(['step', 'stacking_prediction_onset', 'stacking_prediction_wakeup'])\n# # padding部分を削除\n# df = df.filter(pl.col('step') != -1)\n# df = df.group_by(['series_id', 'step']).agg(pl.col(['stacking_prediction_onset', 'stacking_prediction_wakeup']).mean())\n# df = df.with_columns(pl.col(['stacking_prediction_onset', 'stacking_prediction_wakeup']).map_elements(sigmoid))\n# df = df.sort(['series_id', 'step'])\n# dfs.append(df)\n# dfs = pl.concat(dfs)\n# dfs = dfs.group_by(['series_id', 'step']).agg(pl.col(['stacking_prediction_onset', 'stacking_prediction_wakeup']).mean())\n# dfs = dfs.with_columns(pl.col('step').cast(pl.Int64))\n# dfs.write_parquet(shimacos_output_dir / f'{model_name}_result.parquet')\n\n ","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:52:19.78965Z","iopub.execute_input":"2023-12-05T11:52:19.789982Z","iopub.status.idle":"2023-12-05T11:52:34.805512Z","shell.execute_reply.started":"2023-12-05T11:52:19.789956Z","shell.execute_reply":"2023-12-05T11:52:34.804345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## sakami stacking","metadata":{}},{"cell_type":"code","source":"# %%python\n# from __future__ import annotations\n\n# import gc\n# from runpy import run_path\n\n# import numpy as np\n# import pandas as pd\n# import polars as pl\n# import torch\n# from torch.utils.data import DataLoader\n\n# from settings import output_dir, shimacos_output_dir, stacking_dir1, stacking_dir2\n# from utils import timer\n\n# model_dir1 = stacking_dir1 / \"models\"\n# model_dir2 = stacking_dir2 / \"models\"\n\n# device = \"cuda\"\n# batch_size = 32\n\n# module1 = run_path(str(next(stacking_dir1.glob(\"*.py\"))))\n# config1 = module1[\"config\"]\n# SleepDataset1 = module1[\"SleepDataset\"]\n# pad_sequence1 = module1[\"pad_sequence\"]\n# SleepModel1 = module1[\"SleepModel\"]\n# predict1 = module1[\"predict\"]\n\n# module2 = run_path(str(next(stacking_dir2.glob(\"*.py\"))))\n# config2 = module2[\"config\"]\n# SleepDataset2 = module2[\"SleepDataset\"]\n# pad_sequence2 = module2[\"pad_sequence\"]\n# SleepModel2 = module2[\"SleepModel\"]\n# predict2 = module2[\"predict\"]\n\n# with timer(\"load data\"):\n# df: pl.DataFrame = pd.read_pickle(shimacos_output_dir / \"009_stacking_data.pkl\")\n\n# with timer(\"preprocess\"):\n# # split into chunks\n# df = df.with_columns(pl.arange(0, pl.count()).alias(\"row_id\"))\n# series_chunk_ids, _, series_chunk_row_ids = zip(\n# *df.group_by([\"series_id\", \"chunk_id\"]).agg(\"row_id\").rows()\n# )\n# series_chunk_row_ids = [np.array(series_row_ids) for series_row_ids in series_chunk_row_ids]\n\n# # drop features\n# assert config1.categorical_columns[0] == \"minute\"\n\n# numerical_inputs = df.select(config1.numerical_columns).fill_null(0.0).to_numpy().astype(np.float32)\n# categorical_inputs = df.select(config1.categorical_columns).fill_null(0.0).to_numpy().astype(np.int64) + 1\n\n# row_ids = df[\"row_id\"]\n# del df\n# gc.collect()\n\n# with timer(\"predict model 1\"):\n# targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n# test_dataset = SleepDataset1(\n# series_chunk_row_ids=series_chunk_row_ids,\n# numerical_inputs=numerical_inputs,\n# categorical_inputs=categorical_inputs,\n# targets=targets,\n# )\n# test_loader = DataLoader(\n# test_dataset,\n# batch_size=batch_size,\n# shuffle=False,\n# drop_last=False,\n# collate_fn=pad_sequence1,\n# num_workers=4,\n# pin_memory=True,\n# )\n\n# model = SleepModel1(\n# numerical_input_size=numerical_inputs.shape[1],\n# categorical_input_size=categorical_inputs.shape[1],\n# )\n# model.eval()\n# model.to(device)\n\n# prediction_values = np.zeros((len(numerical_inputs), 2), dtype=np.float32)\n# model_paths = list(model_dir1.glob(\"model_fold*.pth\"))\n# for model_path in model_paths:\n# model.load_state_dict(torch.load(model_path, map_location=device))\n# pred_df = predict1(model, test_loader).filter(pl.col(\"row_id\") != -1)\n# assert (pred_df[\"row_id\"] == row_ids).all()\n# prediction_values[:, 0] += pred_df[\"stacking_prediction_onset\"].to_numpy() / len(model_paths)\n# prediction_values[:, 1] += pred_df[\"stacking_prediction_wakeup\"].to_numpy() / len(model_paths)\n\n# del test_dataset, test_loader\n# gc.collect()\n\n# pl.DataFrame(\n# {\n# \"row_id\": row_ids,\n# \"stacking_prediction_onset\": prediction_values[:, 0],\n# \"stacking_prediction_wakeup\": prediction_values[:, 1],\n# }\n# ).write_parquet(output_dir / \"004_transformer_preds.parquet\")\n\n# with timer(\"predict model 2\"):\n# targets = np.zeros((len(numerical_inputs), 1), dtype=np.float32)\n# test_dataset = SleepDataset2(\n# series_chunk_row_ids=series_chunk_row_ids,\n# numerical_inputs=numerical_inputs,\n# categorical_inputs=categorical_inputs,\n# targets=targets,\n# )\n# test_loader = DataLoader(\n# test_dataset,\n# batch_size=batch_size,\n# shuffle=False,\n# drop_last=False,\n# collate_fn=pad_sequence2,\n# num_workers=4,\n# pin_memory=True,\n# )\n\n# model = SleepModel2(\n# numerical_input_size=numerical_inputs.shape[1],\n# categorical_input_size=categorical_inputs.shape[1],\n# )\n# model.eval()\n# model.to(device)\n\n# prediction_values = np.zeros((len(numerical_inputs), 2), dtype=np.float32)\n# model_paths = list(model_dir2.glob(\"model_fold*.pth\"))\n# for model_path in model_paths:\n# model.load_state_dict(torch.load(model_path, map_location=device))\n# pred_df = predict2(model, test_loader).filter(pl.col(\"row_id\") != -1)\n# assert (pred_df[\"row_id\"] == row_ids).all()\n# prediction_values[:, 0] += pred_df[\"stacking_prediction_onset\"].to_numpy() / len(model_paths)\n# prediction_values[:, 1] += pred_df[\"stacking_prediction_wakeup\"].to_numpy() / len(model_paths)\n\n# del test_dataset, test_loader\n# gc.collect()\n\n# pl.DataFrame(\n# {\n# \"row_id\": row_ids,\n# \"stacking_prediction_onset\": prediction_values[:, 0],\n# \"stacking_prediction_wakeup\": prediction_values[:, 1],\n# }\n# ).write_parquet(output_dir / \"011_cnn_preds.parquet\")","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:53:15.36644Z","iopub.execute_input":"2023-12-05T11:53:15.366813Z","iopub.status.idle":"2023-12-05T11:53:15.376013Z","shell.execute_reply.started":"2023-12-05T11:53:15.366782Z","shell.execute_reply":"2023-12-05T11:53:15.375076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Weighted Average","metadata":{}},{"cell_type":"code","source":"# %%writefile ensemble_config.py\n\n# shimacos_model_name2weight = {\n# 'stacking_exp028_021_from_016_192step': 0.17974514198498276,\n# 'stacking_exp029_021_from_028_catboost': 0.128142040651809,\n# 'stacking_exp030_021_chink_gru': 0.12650030330772288,\n# }\n# sakami_model_name2weight = {\n# '004_transformer': 0.22507631533063793,\n# '011_cnn': 0.3405361987248475\n# }\n\n# pp_params = {\n# 'daily_score_offset': 3.84,\n# 'event_rate': 500,\n# 'height': 0.001,\n# 'event2offset': {\"onset\": \"5h\", \"wakeup\": \"0h\"},\n# 'use_daily_norm': True,\n# 'later_date_max_sub_rate': None,\n# }","metadata":{"execution":{"iopub.status.busy":"2023-12-05T11:53:16.743488Z","iopub.execute_input":"2023-12-05T11:53:16.744154Z","iopub.status.idle":"2023-12-05T11:53:16.748629Z","shell.execute_reply.started":"2023-12-05T11:53:16.744123Z","shell.execute_reply":"2023-12-05T11:53:16.74759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%python\n\n# import gc\n# from runpy import run_path\n\n# import numpy as np\n# import pandas as pd\n# import polars as pl\n\n# from utils import timer\n\n# from ensemble_config import shimacos_model_name2weight, sakami_model_name2weight, pp_params\n# from settings import output_dir, shimacos_output_dir, stacking_dir1, stacking_dir2\n# from shimacos_utils import post_process_from_2nd\n\n \n\n# name2weight = shimacos_model_name2weight | sakami_model_name2weight\n# print(f\"model weights {name2weight}\")\n\n# def load_sakami_pred(name):\n# print(f\"load {name} pred\")\n# df: pl.DataFrame = pd.read_pickle(shimacos_output_dir / \"009_stacking_data.pkl\").select(['series_id', 'step', 'timestamp'])\n# pred_df = pl.concat(\n# [ \n# df,\n# pl.read_parquet(output_dir / f'{name}_preds.parquet')\n# ], how=\"horizontal\"\n# ).with_columns(\n# pl.col(\"step\").cast(pl.UInt32)\n# ).rename({\n# \"stacking_prediction_onset\": f\"{name}_stacking_prediction_onset\",\n# \"stacking_prediction_wakeup\": f\"{name}_stacking_prediction_wakeup\",\n# })\n# return pred_df.select([\"series_id\", \"step\", f\"{name}_stacking_prediction_onset\", f\"{name}_stacking_prediction_wakeup\"])\n\n# def load_shimacos_pred(name: str):\n# print(f\"load {name} pred\")\n# pred_df = pl.read_parquet(shimacos_output_dir / f'{name}_result.parquet'\n# ).rename({\n# \"stacking_prediction_onset\": f\"{name}_stacking_prediction_onset\",\n# \"stacking_prediction_wakeup\": f\"{name}_stacking_prediction_wakeup\",\n# }).with_columns(pl.col(\"step\").cast(pl.UInt32))\n# return pred_df.select([\"series_id\", \"step\", f\"{name}_stacking_prediction_onset\", f\"{name}_stacking_prediction_wakeup\"])\n\n\n# def load_and_concat_shimacos_preds(train_df):\n# train_df = train_df.with_columns(pl.lit(0).alias(\"count\"))\n\n# for name in shimacos_model_name2weight.keys():\n# pred_df = load_shimacos_pred(name)\n# train_df = train_df.join(pred_df, on=['series_id', 'step'], how='outer')\n# train_df = train_df.with_columns(pl.col(\"count\") + pl.col(f\"{name}_stacking_prediction_onset\").is_not_null().cast(int)) # null でないものの数をカウント\n\n# for name in sakami_model_name2weight.keys():\n# pred_df = load_sakami_pred(name)\n# train_df = train_df.join(pred_df, on=['series_id', 'step'], how='outer')\n# train_df = train_df.with_columns(pl.col(\"count\") + pl.col(f\"{name}_stacking_prediction_onset\").is_not_null().cast(int)) # null でないものの数をカウント \n \n# # countが0のものは除く\n# train_df = train_df.filter(pl.col(\"count\") > 0)\n\n# # null を 0 にする\n# train_df = train_df.fill_null(0.0)\n\n# # 12stepごとにchunk_idを振る\n# train_df = train_df.with_columns(\n# ((pl.col(\"step\") - pl.col(\"step\").shift(1)) != 12)\n# .cast(int)\n# .cumsum()\n# .over(\"series_id\")\n# .fill_null(0)\n# .alias(\"chunk_id\")\n# ).with_columns(pl.col(\"step\").cast(pl.UInt32))\n\n# return train_df\n\n\n# def make_submission(name2weight, params, pred_df):\n# pred_df = pred_df.with_columns(\n# pl.sum_horizontal(\n# [pl.col(f\"{name}_stacking_prediction_onset\") * weight for name, weight in name2weight.items()]).alias(\"stacking_prediction_onset\"),\n# pl.sum_horizontal(\n# [pl.col(f\"{name}_stacking_prediction_wakeup\") * weight for name, weight in name2weight.items()]).alias(\"stacking_prediction_wakeup\"),\n# )\n# sub_df = post_process_from_2nd(\n# pred_df,\n# **params\n# )\n# return sub_df\n\n\n# test_df = pl.read_parquet(output_dir / \"test_series.parquet\", columns=[\"series_id\", \"step\", \"timestamp\"])\n# test_df = test_df.with_columns(pl.col(\"timestamp\").str.to_datetime(\"%Y-%m-%dT%H:%M:%S%z\")).filter(\n# pl.col(\"step\") % 12 == 0\n# ).with_columns(pl.col(\"step\").cast(pl.UInt32))\n# pred_df = load_and_concat_shimacos_preds(test_df)\n\n\n# submission_df = make_submission(name2weight, pp_params, pred_df)\n# print(len(submission_df))\n# submission_df.write_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-12-05T09:19:00.696012Z","iopub.execute_input":"2023-12-05T09:19:00.696301Z","iopub.status.idle":"2023-12-05T09:19:04.018371Z","shell.execute_reply.started":"2023-12-05T09:19:00.696272Z","shell.execute_reply":"2023-12-05T09:19:04.01745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%python\n\nimport polars as pl\nimport numpy as np\n\nfrom settings import shimacos_output_dir\nfrom shimacos_utils import post_process_from_2nd\n\nif __name__ == '__main__':\n model_names = [\n 'stacking_exp059_030_truncate_lgbm',\n 'stacking_exp061_030_truncate_cat',\n ]\n weights = [1, 3]\n pred_df0 = pl.read_parquet(shimacos_output_dir / f'{model_names[0]}_result.parquet')\n pred_df1 = pl.read_parquet(shimacos_output_dir / f'{model_names[1]}_result.parquet')\n # pred_df0の順番に合わせる\n pred_df1 = pred_df0[['series_id', 'step']].join(pred_df１, on=['series_id', 'step'], how='left')\n\n pred_df0 = pred_df0.with_columns(\n pl.Series(\n \"stacking_prediction_onset\",\n np.average(\n [\n pred_df0[\"stacking_prediction_onset\"],\n pred_df1[\"stacking_prediction_onset\"],\n ],\n weights=weights,\n axis=0,\n ),\n ),\n pl.Series(\n \"stacking_prediction_wakeup\",\n np.average(\n [\n pred_df0[\"stacking_prediction_wakeup\"],\n pred_df1[\"stacking_prediction_wakeup\"],\n ],\n weights=weights,\n axis=0,\n ),\n ),\n ).sort([\"series_id\", \"step\"])\n sub_df = post_process_from_2nd(pred_df0, daily_score_offset=7)\n sub_df.write_csv('submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-12-05T12:13:10.541005Z","iopub.execute_input":"2023-12-05T12:13:10.541896Z","iopub.status.idle":"2023-12-05T12:13:14.891439Z","shell.execute_reply.started":"2023-12-05T12:13:10.54186Z","shell.execute_reply":"2023-12-05T12:13:14.890508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm -rf output\n!rm -rf src\n!rm -rf __pycache__\n!rm settings.py\n!rm utils.py","metadata":{"execution":{"iopub.status.busy":"2023-12-05T09:19:04.019536Z","iopub.execute_input":"2023-12-05T09:19:04.019862Z","iopub.status.idle":"2023-12-05T09:19:08.764395Z","shell.execute_reply.started":"2023-12-05T09:19:04.019835Z","shell.execute_reply":"2023-12-05T09:19:08.763139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-12-05T09:19:08.766221Z","iopub.execute_input":"2023-12-05T09:19:08.766973Z","iopub.status.idle":"2023-12-05T09:19:09.747648Z","shell.execute_reply.started":"2023-12-05T09:19:08.766929Z","shell.execute_reply":"2023-12-05T09:19:09.746536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}