{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"mkdir src","metadata":{"execution":{"iopub.status.busy":"2023-11-09T14:12:20.386465Z","iopub.execute_input":"2023-11-09T14:12:20.386884Z","iopub.status.idle":"2023-11-09T14:12:21.533991Z","shell.execute_reply.started":"2023-11-09T14:12:20.386852Z","shell.execute_reply":"2023-11-09T14:12:21.532456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/__init__.py\n\nfrom pathlib import Path\nfrom types import SimpleNamespace\n\nmeta_config = SimpleNamespace()\nmeta_config.input_dir = Path(\".\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-09T14:12:21.536469Z","iopub.execute_input":"2023-11-09T14:12:21.536848Z","iopub.status.idle":"2023-11-09T14:12:21.544200Z","shell.execute_reply.started":"2023-11-09T14:12:21.536815Z","shell.execute_reply":"2023-11-09T14:12:21.542941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/common.py\n\nimport logging\nimport os\nimport random\nimport shutil\nimport sys\nfrom typing import Type\n\nimport nbclient\nimport nbconvert\nimport nbformat\nimport nbparameterise\nimport numpy as np\nimport torch\nimport wandb\nfrom loguru import logger\nfrom nbconvert import HTMLExporter\n\nfrom . import meta_config\nfrom .utils import clear_memory, timer, upload_to_gcs\n\n\ndef setup(config: Type) -> None:\n    # wandb configuration\n    os.environ[\"WANDB_API_KEY\"] = meta_config.wandb_api_key\n    wandb.init(project=meta_config.wandb_project, name=meta_config.run_name, reinit=True)\n\n    config_vars = {}\n    for key in dir(config):\n        if key.startswith(\"_\"):\n            continue\n        value = getattr(config, key)\n        if isinstance(value, dict):\n            for k, v in value.items():\n                config_vars[f\"{key}_{k}\"] = v\n        else:\n            config_vars[key] = value\n\n    wandb.config.update(config_vars)\n\n    # set seed\n    random.seed(config.seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(config.seed)\n    np.random.seed(config.seed)\n    torch.manual_seed(config.seed)\n    torch.cuda.manual_seed(config.seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n    # setup logger\n    logger.remove()\n    logger.add(sys.stdout, level=logging.INFO)\n    logger.add(meta_config.save_dir / \"log.txt\", mode=\"w\", level=logging.DEBUG)\n\n    # log config\n    logger.info(\"configuration params:\")\n    for key in dir(config):\n        if key.startswith(\"_\"):\n            continue\n\n        value = getattr(config, key)\n        if isinstance(value, list):\n            logger.info(f\"\\t {key}:\")\n            for v in value:\n                logger.info(f\"\\t\\t {v}\")\n        elif isinstance(value, dict):\n            logger.info(f\"\\t {key}:\")\n            for k, v in value.items():\n                logger.info(f\"\\t\\t {k}: {v}\")\n        else:\n            logger.info(f\"\\t {key}: {value}\")\n\n\n@clear_memory\n@timer()\ndef report_result(report_idx: int = 0, debug: bool = False) -> None:\n    \"\"\"Report the execution result.\n    The output format is HTML.\n\n    Parameters\n    ----------\n    report_idx : int, default 0\n        ID of the report notebook.\n    \"\"\"\n    with open(meta_config.home_dir / \"report/\" / f\"report{report_idx}.ipynb\") as f:\n        nb = nbformat.read(f, as_version=4)\n\n    params = nbparameterise.parameter_values(\n        nbparameterise.extract_parameters(nb), output_dir=str(meta_config.save_dir), debug=debug\n    )\n    updated_nb = nbparameterise.replace_definitions(nb, params)\n    nbclient.execute(updated_nb)\n    body, _ = nbconvert.export(HTMLExporter, updated_nb)\n    (meta_config.save_dir / \"report.html\").write_text(body)\n\n\ndef finish() -> None:\n    # report\n    wandb.log({\"report\": wandb.Html(open(meta_config.save_dir / \"report.html\"))})\n    report_url = (\n        f\"https://storage.cloud.google.com/{meta_config.gcp_bucket}/artifacts/{meta_config.run_name}/report.html\"\n    )\n    wandb.run.summary.update({\"report_url\": report_url})\n\n    # finish wandb\n    wandb.finish()\n\n    # upload artifacts to GCS\n    upload_to_gcs(meta_config.save_dir, gcs_prefix=\"sakami/artifacts/\")\n\n    # avoid `No space left on device` error\n    shutil.rmtree(meta_config.wandb_dir)\n    shutil.rmtree(meta_config.save_dir)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/config.py\n\nfrom __future__ import annotations\n\nimport copy\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\nimport yaml\n\n\nclass Config(dict):\n    \"\"\"A dictionary class that allows for attribute-style access of values.\"\"\"\n\n    __setattr__ = dict.__setitem__\n\n    def init(self) -> None:\n        # update meta_config\n        exec_file_name = sys.argv[0]\n\n        self.home_dir = Path(\"./\")\n        self.input_dir = Path(\"./input\")\n        self.data_dir = self.input_dir / self.competition_name\n        self.cache_dir = self.input_dir / \"cache\"\n        self.output_dir = Path(\"./output\")\n        self.wandb_dir = Path(\"./wandb\")\n\n        self.run_name = Path(exec_file_name).stem\n        self.save_dir = Path(self.output_dir) / self.run_name\n        self.checkpoint_dir = self.save_dir / \"checkpoint/\"\n\n        # make directories\n        self.input_dir.mkdir(exist_ok=True)\n        self.data_dir.mkdir(exist_ok=True)\n        self.cache_dir.mkdir(exist_ok=True)\n        self.output_dir.mkdir(exist_ok=True)\n        self.wandb_dir.mkdir(exist_ok=True)\n        self.save_dir.mkdir(exist_ok=True)\n        self.checkpoint_dir.mkdir(exist_ok=True)\n\n    def check_columns(self) -> None:\n        required_columns = [\n            \"competition_name\",\n            \"gcp_user_name\",\n            \"gcp_project\",\n            \"gcp_bucket\",\n            \"line_token\",\n            \"wandb_api_key\",\n            \"wandb_project\",\n        ]\n        for required_column in required_columns:\n            if not hasattr(self, required_column):\n                raise KeyError(f\"Meta config {required_column} must be specified.\")\n\n    def __getattr__(self, key: Any) -> Any:\n        value = super().get(key)\n        if isinstance(value, dict):\n            return Config(value)\n        return value\n\n    def __deepcopy__(self, memo: dict[int, int | list[int]] | None = None) -> Config:\n        \"\"\"Prevent errors in the `copy.deepcopy` method.\n\n        References\n        ----------\n        - https://stackoverflow.com/questions/49901590/python-using-copy-deepcopy-on-dotdict\n        \"\"\"\n        return Config(copy.deepcopy(dict(self), memo=memo))\n\n    @classmethod\n    def load(cls, config_path: str) -> Config:\n        \"\"\"Load a config file.\n\n        Parameters\n        ----------\n        config_path : str\n            Path to config file.\n\n        Returns\n        -------\n        Config\n            Configuration parameters.\n        \"\"\"\n        if not Path(config_path).exists():\n            raise ValueError(f\"Configuration file {config_path} does not exist.\")\n\n        with open(config_path) as f:\n            config = cls(yaml.safe_load(f))\n\n        config.check_columns()\n        config.init()\n        return config","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/detect_peak.py\n\nimport numpy as np\nimport polars as pl\nfrom numba import jit\nfrom tqdm.auto import tqdm\n\n\ndef post_process_from_2nd(\n    pred_df,\n    event_rate: int | float = 500,\n    height: float = 0.001,\n    event2col: dict[str, str] = {\"onset\": \"stacking_prediction_onset\", \"wakeup\": \"stacking_prediction_wakeup\"},\n    event2offset: dict[str, str] = {\"onset\": \"5h\", \"wakeup\": \"0h\"},\n    use_daily_norm: bool = True,\n    daily_score_offset: int = 10,\n    later_date_max_sub_rate: float | None = 0.05,\n):\n    \"\"\"\n    1分ごとの予測値を用いてイベントを検出する\n    用語\n    - 予測地点: 2段目のモデルによって得られた1分毎の予測位置\n    - 候補地点: event の候補となる 15秒 or 45秒始まりの30秒間隔の位置\n\n    Args:\n        pred_df (pl.DataFrame): timestamp 込み\n        event_rate (int | float, optional): [0,1) の値であれば1分間に何回イベントが起こるか。intの場合はseries_idごとに同じイベント数を検出。 Defaults to 0.005.\n        height (float, optional): 候補地点の期待値がこの値を下回ったら終了。 Defaults to 0.1.\n        event2col (dict[str, str], optional): event名と予測値のカラム名の対応。 Defaults to {\"onset\": \"stacking_prediction_onset\", \"wakeup\": \"stacking_prediction_wakeup\"}.\n        weight_rate (float | None, optional): 遠くの予測値の期待値を割り引く際の重み。Noneの場合は重みを1とする。1/weight_rate 倍ずつ遠くの予測値の重みが小さくなっていく。 Defaults to None.\n        use_daily_norm (bool, optional): 一日ごとに予測値を正規化するかどうか。 Defaults to False.\n        daily_score_offset (float, optional): 正規化の際のoffset。 Defaults to 1.0.\n        later_date_max_sub_rate (float | None, optional): 日付が古いほど予測値を割り引く際の最大割引率。Noneの場合は割引しない。 Defaults to None.\n    Returns:\n        event_df (pl.DataFrame): row_id, series_id, step, event, score をカラムに持つ。\n    \"\"\"\n    high_match_nums = (0, 1, 3, 5, 8, 10, 13, 15, 20, 25, 30)\n    low_match_nums = (0, 1, 3, 5, 7, 10, 12, 15, 20, 25, 30)\n    match_sums = np.ones(10)\n    result_events_records = []\n\n    # event ごとに処理\n    for event, event_pred_col in event2col.items():\n        \"\"\"\n        元の系列の予測地点(長さN): 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, ..., (N-1)*12\n        15秒から30秒おきのevent候補地点(長さ2N): 3, 9, 15, 21, 27, 33, 39, 45, 51, 57, ..., (N-1)*12+3, (N-1)*12+9\n            - 15秒(3step)から1分おき(長さN): 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n                - 左の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums\n                - 右の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n            - 45秒(9step)から1分おき(長さN): 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n                - 左の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n                - 右の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums       \n        \"\"\"\n\n        # series内でのindexを振り、chunk内での最大と最小を計算\n        minute_pred_df = pred_df.filter(pl.col(\"timestamp\").is_not_null())\n\n        if use_daily_norm:\n            minute_pred_df = (\n                minute_pred_df.with_columns(\n                    pl.col(\"timestamp\").dt.offset_by(event2offset[event]).dt.date().alias(\"date\")\n                )\n                .with_columns(pl.col(event_pred_col).sum().over([\"series_id\", \"date\"]).alias(\"date_sum\"))\n                .with_columns(\n                    pl.col(event_pred_col) / (pl.col(\"date_sum\") + (1 / (daily_score_offset + pl.col(\"date_sum\"))))\n                )\n            )\n        if later_date_max_sub_rate is not None:\n            minute_pred_df = minute_pred_df.with_columns(\n                pl.col(\"date\").min().over(\"series_id\").alias(\"min_date\"),\n                pl.col(\"date\").max().over(\"series_id\").alias(\"max_date\"),\n            ).with_columns(\n                pl.col(event_pred_col)\n                * (\n                    pl.lit(1.0)\n                    - (\n                        pl.lit(later_date_max_sub_rate)\n                        * (\n                            (pl.col(\"date\") - pl.col(\"min_date\")).dt.days().cast(float)\n                            / ((pl.col(\"max_date\") - pl.col(\"min_date\")).dt.days().cast(float) + 1.0)\n                        )\n                    )\n                )\n            )\n\n        max_event_per_series = event_rate if isinstance(event_rate, int) else int(len(minute_pred_df) * event_rate)\n\n        # series_id, chunk_id, step でソート\n        minute_pred_df = minute_pred_df.sort([\"series_id\", \"chunk_id\", \"step\"])\n\n        # 1. 期待値の計算\n        # 1.1 左側を計算 (同じindexの予測を含む左側を計算)\n        \"\"\"\n        以下をそれぞれ計算する\n        - 15秒(3step)から1分おき(長さN)での候補地点での期待値: stepは 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n        - 45秒(9step)から1分おき(長さN)での候補地点での期待値: stepは 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n        計算は左側の予測地点の数と、右側の予測地点の数\n        \"\"\"\n        minute_pred_df = minute_pred_df.with_columns(\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over([\"series_id\", \"chunk_id\"])\n                        / match_sums[i]\n                    )\n                    for i, window in enumerate(high_match_nums[1:])\n                ]\n            ).alias(f\"{event}_left_expectation_plus_3step\"),\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over([\"series_id\", \"chunk_id\"])\n                        / match_sums[i]\n                    )\n                    for i, window in enumerate(low_match_nums[1:])\n                ]\n            ).alias(f\"{event}_left_expectation_plus_9step\"),\n        )\n\n        # 1.2 右側を計算(同じindexの予測を含まない右側を計算。逆順にして一個ずらしrolling_sumを取る必要がある）\n        minute_pred_df = minute_pred_df.reverse()\n        minute_pred_df = minute_pred_df.with_columns(\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .shift(1)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over([\"series_id\", \"chunk_id\"])\n                        .fill_null(0)\n                        / match_sums[i]\n                    )\n                    for i, window in enumerate(low_match_nums[1:])\n                ]\n            ).alias(f\"{event}_right_expectation_plus_3step\"),\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .shift(1)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over([\"series_id\", \"chunk_id\"])\n                        .fill_null(0)\n                        / match_sums[i]\n                    )\n                    for i, window in enumerate(high_match_nums[1:])\n                ]\n            ).alias(f\"{event}_right_expectation_plus_9step\"),\n        )\n        minute_pred_df = minute_pred_df.reverse()\n\n        # 合計の期待値計算\n        minute_pred_df = minute_pred_df.with_columns(\n            (pl.col(f\"{event}_left_expectation_plus_3step\") + pl.col(f\"{event}_right_expectation_plus_3step\")).alias(\n                f\"{event}_expectation_sum_3step\"\n            ),\n            (pl.col(f\"{event}_left_expectation_plus_9step\") + pl.col(f\"{event}_right_expectation_plus_9step\")).alias(\n                f\"{event}_expectation_sum_9step\"\n            ),\n        )\n\n        # print(display(minute_pred_df))\n\n        # 3. 最大値の取得 & 期待値の割引\n        \"\"\"\n        各予測地点の power を管理する。powerは以下の11種類\n        0: その予測地点が影響を与える範囲は無い\n        1: その予測地点が影響を与える範囲は左右1つ(1min)\n        2: その予測地点が影響を与える範囲は左右3つ\n        ︙\n        10: 左右30(step 0~360)\n\n        event を作るたびに、eventからtolerance内にある予測地点のpowerを下げる。\n        その際に予測地点からtolerance内にある、eventがあったところも含めた候補地点の期待値を割り引く。\n        \"\"\"\n        for series_id, series_df in tqdm(\n            minute_pred_df.select(\n                [\n                    \"series_id\",\n                    \"chunk_id\",\n                    \"step\",\n                    event_pred_col,\n                    f\"{event}_expectation_sum_3step\",\n                    f\"{event}_expectation_sum_9step\",\n                ]\n            ).group_by(\"series_id\"),\n            desc=f\"detect {event} peaks\",\n            leave=False,\n            total=len(minute_pred_df[\"series_id\"].unique()),\n        ):\n            # chunkごとの id の最大最小を計算\n            series_df = series_df.with_row_count().with_columns(\n                pl.col(\"row_nr\").max().over([\"chunk_id\"]).alias(\"max_id_in_chunk\"),\n                pl.col(\"row_nr\").min().over([\"chunk_id\"]).alias(\"min_id_in_chunk\"),\n            )\n\n            preds = series_df[event_pred_col].to_numpy()\n            expectation_sum_3step = series_df[f\"{event}_expectation_sum_3step\"].to_numpy(writable=True)\n            expectation_sum_9step = series_df[f\"{event}_expectation_sum_9step\"].to_numpy(writable=True)\n            steps = series_df[\"step\"].to_numpy(writable=True)\n            step_id_mins = series_df[\"min_id_in_chunk\"].to_numpy(writable=True)\n            step_id_maxs = series_df[\"max_id_in_chunk\"].to_numpy(writable=True) + 1\n            powers = np.ones(len(expectation_sum_3step), dtype=np.int32) * 10\n\n            result_steps, result_scores = detect_events_for_serie(\n                height,\n                max_event_per_series,\n                high_match_nums,\n                low_match_nums,\n                match_sums,\n                steps,\n                step_id_mins,\n                step_id_maxs,\n                preds,\n                expectation_sum_3step,\n                expectation_sum_9step,\n                powers,\n            )\n\n            for i in range(len(result_steps)):\n                result_events_records.append(\n                    {\n                        \"series_id\": series_id,\n                        \"step\": result_steps[i],\n                        \"event\": event,\n                        \"score\": result_scores[i],\n                    }\n                )\n\n            # print(expectation_sum_3step[max_step_index], expectation_sum_9step[max_step_index])\n\n    if len(result_events_records) == 0:  # 一つも予測がない場合はdummyを入れる\n        result_events_records.append(\n            {\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            }\n        )\n    sub_df = pl.DataFrame(result_events_records).sort(by=[\"series_id\", \"step\"])\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df\n\n\n@jit(nopython=True, cache=True)\ndef detect_events_for_serie(\n    height,\n    max_event_per_series,\n    high_match_nums,\n    low_match_nums,\n    match_sums,\n    steps,\n    step_id_mins,\n    step_id_maxs,\n    preds,\n    expectation_sum_3step,\n    expectation_sum_9step,\n    powers,\n):\n    result_steps = []\n    result_scores = []\n    for _ in range(max_event_per_series):  # 高い順に最大max_event_per_series個のeventを決定\n        # 3.1 最大値の取得\n        # 合計の期待値が最大のstepを取得\n        max_step3 = expectation_sum_3step.argmax()\n        max_score3 = expectation_sum_3step[max_step3]\n        max_step9 = expectation_sum_9step.argmax()\n        max_score9 = expectation_sum_9step[max_step9]\n        if max_score3 > max_score9:\n            # print('max_score3')\n            left_nums = high_match_nums\n            right_nums = low_match_nums\n            max_step_index = max_step3\n            max_score = max_score3\n            if max_score < height:  # 閾値以下なら終了\n                break\n            result_steps.append(steps[max_step_index] + 3)\n            result_scores.append(max_score)\n        else:\n            # print('max_score9')\n            left_nums = low_match_nums\n            right_nums = high_match_nums\n            max_step_index = max_step9\n            max_score = max_score9\n            if max_score < height:  # 閾値以下なら終了\n                break\n            result_steps.append(steps[max_step_index] + 9)\n            result_scores.append(max_score)\n        # print(f\"max_step_index:{max_step_index}, max_score:{max_score}\")\n\n        # 3.2 期待値の割引\n        \"\"\"\n        各予測地点のpowerを修正するとともに、候補地点の期待値を割引く。\n        powerが pi まで小さくなることによってその予測値が影響を与える範囲が狭くなる。\n        つまり狭くなって範囲から抜けた expectation_sum の値が、その予測値の値*重みの分だけ小さくなる\n        \"\"\"\n        # 3.2.1 まずはpowerを修正するstepの候補を探す\n        target_step_powers = []  # (target_step, pred, base_power, power, step_min, step_max)のリスト\n        for pi in range(0, 10):\n            # 左側\n            for l_diff in range(left_nums[pi], left_nums[pi + 1]):\n                target_step_index = max_step_index - l_diff\n                if target_step_index < 0:\n                    break\n                pred = preds[target_step_index]\n                base_power = powers[target_step_index]\n                if base_power > pi:  # power が小さくなる場合のみ修正\n                    target_step_powers.append(\n                        (\n                            target_step_index,\n                            pred,\n                            base_power,\n                            pi,\n                            step_id_mins[target_step_index],\n                            step_id_maxs[target_step_index],\n                        )\n                    )\n            # 右側\n            for r_diff in range(right_nums[pi] + 1, right_nums[pi + 1] + 1):  # 自分自身と同じindexは含めない\n                target_step_index = max_step_index + r_diff\n                if target_step_index >= len(powers):\n                    break\n                pred = preds[target_step_index]\n                base_power = powers[target_step_index]\n                if base_power > pi:\n                    target_step_powers.append(\n                        (\n                            target_step_index,\n                            pred,\n                            base_power,\n                            pi,\n                            step_id_mins[target_step_index],\n                            step_id_maxs[target_step_index],\n                        )\n                    )\n        # print('target_step_powers', target_step_powers)\n\n        # 3.2.2 対象となる step の power を修正するとともに期待値を割り引く\n        \"\"\"\n        予測地点のpowerを下げるとともに、関連する候補地点の期待値を修正する。\n        検出したeventから遠い予測地点の場合は、予測地点に近い候補地点であっても期待値はその分割り引かれる。\n        3stepの修正をする時は target_stepから左側が low_match_nums, 右側が high_match_nums\n        9stepの修正をする時は target_stepから左側が high_match_nums, 右側が low_match_nums\n        だんだんと内側のみがのこるように修正する。\n\n        - powerが 10 → 8 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~20個に影響を及ぼすようになる。また、powerが2個減った分全体の期待値も割り引かれる\n        - powerが 10 → 5 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~12(13)個に影響を及ぼすようになる\n        - powerが 8 → 7 になるケースは左右1~20個に影響を及ぼしていたものが、左右の1~15個に影響を及ぼすようになる\n        \"\"\"\n        for si, pred, base_power, power, step_min, step_max in target_step_powers:\n            # print(f\"si:{si}, pred:{pred}, base_power:{base_power}, power:{power}\")\n            powers[si] = power\n            # 中心ほど重みが強いので power ごとに処理\n            for pi in range(base_power, power, -1):  # base_powerからpowerに減らしていくことで予測値の外側から削る\n                # 3step\n                left_diff_max = low_match_nums[pi]\n                right_diff_max = high_match_nums[pi]\n                expectation_sum_3step[max(si - left_diff_max, step_min) : min(si + right_diff_max, step_max)] -= (\n                    pred / match_sums[pi - 1]\n                )\n                \"\"\"\n                if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n                    print(f'3step pi:{pi}')\n                    print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\") \n                    print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n                    print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n                    print()\n                \"\"\"\n                # 9step\n                left_diff_max = high_match_nums[pi]\n                right_diff_max = low_match_nums[pi]\n                expectation_sum_9step[max(si - left_diff_max, step_min) : min(si + right_diff_max, step_max)] -= (\n                    pred / match_sums[pi - 1]\n                )\n                \"\"\"\n                if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n                    print(f'9step pi:{pi}')\n                    print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\") \n                    print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n                    print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n                    print()\n                \"\"\"\n    return result_steps, result_scores\n\n\ndef remove_periodicity(\n    df,\n    periodicity_dict,\n    event2col: dict[str, str] = {\"onset\": \"prediction_onset\", \"wakeup\": \"prediction_wakeup\"},\n):\n    \"\"\"\n    雑にdfからperiodicityを削除する\n    \"\"\"\n    dfs = []\n    for series_id, series_df in df.group_by(\"series_id\"):\n        for event, event_pred_col in event2col.items():\n            series_df.with_columns(\n                pl.Series(\n                    name=event_pred_col,\n                    values=series_df.get_column(event_pred_col).to_numpy() * (1 - periodicity_dict[series_id]),\n                )\n            )\n        dfs.append(series_df)\n    return pl.concat(dfs)\n\n\ndef post_process(\n    pred_df,\n    event_rate: int | float = 0.005,\n    height: float = 0.1,\n    event2col: dict[str, str] = {\"onset\": \"prediction_onset\", \"wakeup\": \"prediction_wakeup\"},\n):\n    \"\"\"\n    1分ごとの予測値を用いてイベントを検出する\n    # TODO: 1段目のモデルを入れる場合は1分ごとの予測値をそのまま使わずに周辺の予測値をrollする方が良さそう？\n\n    用語\n    - 予測地点: 1段目 or 2段目のモデルによって得られた1分毎の予測位置\n    - 候補地点: event の候補となる 15秒 or 45秒始まりの30秒間隔の位置\n\n    Args:\n        pred_df (pl.DataFrame): series_id, step, event2colのvalue をカラムに持つ。stepが12の倍数以外の行は無視される。periodicityは削除済み想定\n        event_rate (int | float, optional): [0,1) の値であれば1分間に何回イベントが起こるか。intの場合はseries_idごとに同じイベント数を検出。 Defaults to 0.005.\n        height (float, optional): 候補地点の期待値がこの値を下回ったら終了。 Defaults to 0.1.\n    Returns:\n        event_df (pl.DataFrame): row_id, series_id, step, event, score をカラムに持つ。\n    \"\"\"\n    high_match_nums = [1, 3, 5, 8, 10, 13, 15, 20, 25, 30]\n    low_match_nums = [1, 3, 5, 7, 10, 12, 15, 20, 25, 30]\n    # match_sums = [2, 6, 10, 15, 20, 25, 30, 40, 50, 60]\n    # total_num = sum(high_match_nums + low_match_nums)\n    result_events_records = []\n\n    # event ごとに処理\n    for event, event_pred_col in event2col.items():\n        \"\"\"\n        元の系列の予測地点(長さN): 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, ..., (N-1)*12\n        15秒から30秒おきのevent候補地点(長さ2N): 3, 9, 15, 21, 27, 33, 39, 45, 51, 57, ..., (N-1)*12+3, (N-1)*12+9\n            - 15秒(3step)から1分おき(長さN): 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n                - 左の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums\n                - 右の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n            - 45秒(9step)から1分おき(長さN): 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n                - 左の個数 {12: 1, 36: 3, 60: 5, 90: *7*, 120: 10, 150: *12*, 180: 15, 240: 20, 300: 25, 360: 30} low_match_nums\n                - 右の個数 {12: 1, 36: 3, 60: 5, 90: *8*, 120: 10, 150: *13*, 180: 15, 240: 20, 300: 25, 360: 30} high_match_nums       \n        \"\"\"\n\n        # 12の倍数以外の行は無視して1分毎の予測値にする\n        minute_pred_df = pred_df.filter(pl.col(\"step\") % 12 == 0)\n        max_event_per_series = event_rate if isinstance(event_rate, int) else int(len(minute_pred_df) * event_rate)\n\n        # series_id, step でソート\n        minute_pred_df = minute_pred_df.sort([\"series_id\", \"step\"])\n\n        # 1. 期待値の計算\n        # 1.1 左側を計算 (同じindexの予測を含む左側を計算)\n        \"\"\"\n        以下をそれぞれ計算する\n        - 15秒(3step)から1分おき(長さN)での候補地点での期待値: stepは 3, 15, 27, 39, 51, 63, 75, 87, 99, 111, ..., (N-1)*12+3\n        - 45秒(9step)から1分おき(長さN)での候補地点での期待値: stepは 9, 21, 33, 45, 57, 69, 81, 93, 105, 117, ..., (N-1)*12+9\n        計算は左側の予測地点の数と、右側の予測地点の数\n        \"\"\"\n        minute_pred_df = minute_pred_df.with_columns(\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over(\"series_id\")\n                    )\n                    for window in high_match_nums\n                ]\n            ).alias(f\"{event}_left_expectation_plus_3step\"),\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over(\"series_id\")\n                    )\n                    for window in low_match_nums\n                ]\n            ).alias(f\"{event}_left_expectation_plus_9step\"),\n        )\n\n        # 1.2 右側を計算(同じindexの予測を含まない右側を計算。逆順にして一個ずらしrolling_sumを取る必要がある）\n        minute_pred_df = minute_pred_df.reverse()\n        minute_pred_df = minute_pred_df.with_columns(\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .shift(1)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over(\"series_id\")\n                        .fill_null(0)\n                    )\n                    for window in low_match_nums\n                ]\n            ).alias(f\"{event}_right_expectation_plus_3step\"),\n            pl.sum_horizontal(\n                [\n                    (\n                        pl.col(event_pred_col)\n                        .shift(1)\n                        .rolling_sum(window_size=window, center=False, min_periods=1)\n                        .over(\"series_id\")\n                        .fill_null(0)\n                    )\n                    for window in high_match_nums\n                ]\n            ).alias(f\"{event}_right_expectation_plus_9step\"),\n        )\n        minute_pred_df = minute_pred_df.reverse()\n\n        # 合計の期待値計算\n        minute_pred_df = minute_pred_df.with_columns(\n            (pl.col(f\"{event}_left_expectation_plus_3step\") + pl.col(f\"{event}_right_expectation_plus_3step\")).alias(\n                f\"{event}_expectation_sum_3step\"\n            ),\n            (pl.col(f\"{event}_left_expectation_plus_9step\") + pl.col(f\"{event}_right_expectation_plus_9step\")).alias(\n                f\"{event}_expectation_sum_9step\"\n            ),\n        )\n\n        # 3. 最大値の取得 & 期待値の割引\n        \"\"\"\n        各予測地点の power を管理する。powerは以下の11種類\n        0: その予測地点が影響を与える範囲は無い\n        1: その予測地点が影響を与える範囲は左右1つ(1min)\n        2: その予測地点が影響を与える範囲は左右3つ\n        ︙\n        10: 左右30(step 0~360)\n\n        event を作るたびに、eventからtolerance内にある予測地点のpowerを下げる。\n        その際に予測地点からtolerance内にある、eventがあったところも含めた候補地点の期待値を割り引く。\n        \"\"\"\n        for series_id, series_df in tqdm(\n            minute_pred_df.select(\n                [\"series_id\", event_pred_col, f\"{event}_expectation_sum_3step\", f\"{event}_expectation_sum_9step\"]\n            ).group_by(\"series_id\"),\n            desc=\"find peaks\",\n            leave=False,\n            total=len(minute_pred_df[\"series_id\"].unique()),\n        ):\n            preds = series_df[event_pred_col].to_numpy()\n            expectation_sum_3step = series_df[f\"{event}_expectation_sum_3step\"].to_numpy(writable=True)\n            expectation_sum_9step = series_df[f\"{event}_expectation_sum_9step\"].to_numpy(writable=True)\n            powers = np.ones(len(expectation_sum_3step), dtype=np.int32) * 10\n            for _ in range(max_event_per_series):  # 高い順に最大max_event_per_series個のeventを決定\n                # 3.1 最大値の取得\n                # 合計の期待値が最大のstepを取得\n                max_step3 = expectation_sum_3step.argmax()\n                max_score3 = expectation_sum_3step[max_step3]\n                max_step9 = expectation_sum_9step.argmax()\n                max_score9 = expectation_sum_9step[max_step9]\n                if max_score3 > max_score9:\n                    # print('max_score3')\n                    left_nums = [0] + high_match_nums\n                    right_nums = [0] + low_match_nums\n                    max_step_index = max_step3\n                    max_score = max_score3\n                    result_events_records.append(\n                        {\n                            \"series_id\": series_id,\n                            \"step\": max_step_index * 12 + 3,\n                            \"event\": event,\n                            \"score\": max_score,\n                        }\n                    )\n                else:\n                    # print('max_score9')\n                    left_nums = [0] + low_match_nums\n                    right_nums = [0] + high_match_nums\n                    max_step_index = max_step9\n                    max_score = max_score9\n                    result_events_records.append(\n                        {\n                            \"series_id\": series_id,\n                            \"step\": max_step_index * 12 + 9,\n                            \"event\": event,\n                            \"score\": max_score,\n                        }\n                    )\n                if max_score < height:  # 閾値以下なら終了\n                    break\n                # print(f\"max_step_index:{max_step_index}, max_score:{max_score}\")\n\n                # 3.2 期待値の割引\n                \"\"\"\n                各予測地点のpowerを修正するとともに、候補地点の期待値を割引く。\n                powerが pi まで小さくなることによってその予測値が影響を与える範囲が狭くなる。\n                つまり狭くなって範囲から抜けた expectation_sum の値が、その予測値の値*重みの分だけ小さくなる\n                \"\"\"\n                # 3.2.1 まずはpowerを修正するstepの候補を探す\n                target_step_powers = []  # (target_step, pred, base_power, power)のリスト\n                for pi in range(0, 10):\n                    # 左側\n                    for l_diff in range(left_nums[pi], left_nums[pi + 1]):\n                        target_step_index = max_step_index - l_diff\n                        if target_step_index < 0:\n                            break\n                        pred = preds[target_step_index]\n                        base_power = powers[target_step_index]\n                        if base_power > pi:  # power が小さくなる場合のみ修正\n                            target_step_powers.append((target_step_index, pred, base_power, pi))\n                    # 右側\n                    for r_diff in range(right_nums[pi] + 1, right_nums[pi + 1] + 1):  # 自分自身と同じindexは含めない\n                        target_step_index = max_step_index + r_diff\n                        if target_step_index >= len(powers):\n                            break\n                        pred = preds[target_step_index]\n                        base_power = powers[target_step_index]\n                        if base_power > pi:\n                            target_step_powers.append((target_step_index, pred, base_power, pi))\n                # print('target_step_powers', target_step_powers)\n\n                # 3.2.2 対象となる step の power を修正するとともに期待値を割り引く\n                \"\"\"\n                予測地点のpowerを下げるとともに、関連する候補地点の期待値を修正する。\n                検出したeventから遠い予測地点の場合は、予測地点に近い候補地点であっても期待値はその分割り引かれる。\n                \n                3stepの修正をする時は target_stepから左側が low_match_nums, 右側が high_match_nums\n                9stepの修正をする時は target_stepから左側が high_match_nums, 右側が low_match_nums\n                だんだんと内側のみがのこるように修正する。\n\n                - powerが 10 → 8 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~20個に影響を及ぼすようになる。また、powerが2個減った分全体の期待値も割り引かれる\n                - powerが 10 → 5 になるケースは左右1~30個に影響を及ぼしていたものが、左右の1~12(13)個に影響を及ぼすようになる\n                - powerが 8 → 7 になるケースは左右1~20個に影響を及ぼしていたものが、左右の1~15個に影響を及ぼすようになる\n                \"\"\"\n                # print(expectation_sum_3step[max_step_index], expectation_sum_9step[max_step_index])\n                for si, pred, base_power, power in target_step_powers:\n                    # print(f\"si:{si}, pred:{pred}, base_power:{base_power}, power:{power}\")\n                    powers[si] = power\n                    # 中心ほど重みが強いので power ごとに処理\n                    for pi in range(\n                        base_power, power, -1\n                    ):  # base_powerからpowerに減らしていくことで予測値の外側から削る\n                        # 3step\n                        left_nums = [0] + low_match_nums\n                        right_nums = [0] + high_match_nums\n                        left_diff_max = left_nums[pi]\n                        right_diff_max = right_nums[pi]\n                        expectation_sum_3step[si - left_diff_max : si + right_diff_max] -= pred\n                        \"\"\"\n                        if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n                            print(f'3step pi:{pi}')\n                            print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\") \n                            print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n                            print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n                            print()\n                        \"\"\"\n\n                        # 9step\n                        left_nums = [0] + high_match_nums\n                        right_nums = [0] + low_match_nums\n                        left_diff_max = left_nums[pi]\n                        right_diff_max = right_nums[pi]\n                        expectation_sum_9step[si - left_diff_max : si + right_diff_max] -= pred\n                        \"\"\"\n                        if ((si-left_diff_max <= max_step_index) and (max_step_index < si+right_diff_max)):\n                            print(f'9step pi:{pi}')\n                            print(f\"max_step_index:{max_step_index}, si:{si}, left_diff_max:{left_diff_max}, right_diff_max:{right_diff_max}\") \n                            print(\"[\", si-left_diff_max, si+right_diff_max, \")\")\n                            print(f\"power: {pi}→{pi-1}, pred:{pred}\")\n                            print()\n                        \"\"\"\n\n                # print(expectation_sum_3step[max_step_index], expectation_sum_9step[max_step_index])\n\n    if len(result_events_records) == 0:  # 一つも予測がない場合はdummyを入れる\n        result_events_records.append(\n            {\n                \"series_id\": series_id,\n                \"step\": 0,\n                \"event\": \"onset\",\n                \"score\": 0,\n            }\n        )\n    sub_df = pl.DataFrame(result_events_records).sort(by=[\"series_id\", \"step\"])\n    row_ids = pl.Series(name=\"row_id\", values=np.arange(len(sub_df)))\n    sub_df = sub_df.with_columns(row_ids).select([\"row_id\", \"series_id\", \"step\", \"event\", \"score\"])\n    return sub_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/metric.py\n\n\"\"\"Event Detection Average Precision\n\nAn average precision metric for event detection in time series and\nvideo.\n\n\"\"\"\n\nfrom bisect import bisect_left\nfrom typing import Dict, List, Tuple\n\nimport numpy as np\nimport pandas as pd\nimport pandas.api.types\nimport polars as pl\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\n# Set some placeholders for global parameters\nseries_id_column_name = None\ntime_column_name = None\nevent_column_name = None\nscore_column_name = None\nuse_scoring_intervals = None\nreturn_detections_matched = None\n\n\ndef score(\n    solution: pl.DataFrame,\n    submission: pl.DataFrame,\n    tolerances: Dict[str, List[float]] = {\n        \"onset\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n        \"wakeup\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    },\n    series_id_column_name: str = \"series_id\",\n    time_column_name: str = \"step\",\n    event_column_name: str = \"event\",\n    score_column_name: str = \"score\",\n    use_scoring_intervals: bool = False,\n    return_detections_matched: bool = False,\n) -> float:\n    \"\"\"Event Detection Average Precision, an AUCPR metric for event detection in\n    time series and video.\n\n    This metric is similar to IOU-threshold average precision metrics commonly\n    used in object detection. For events occuring in time series, we replace the\n    IOU threshold with a time tolerance.\n\n    Submissions are evaluated on the average precision of detected events,\n    averaged over timestamp error tolerance thresholds, averaged over event\n    classes.\n\n    Detections are matched to ground-truth events within error tolerances, with\n    ambiguities resolved in order of decreasing confidence.\n\n    Detailed Description\n    --------------------\n    Evaluation proceeds in four steps:\n\n    1. Selection - (optional) Predictions not within a series' scoring\n    intervals are dropped.\n    2. Assignment - Predicted events are matched with ground-truth events.\n    3. Scoring - Each group of predictions is scored against its corresponding\n    group of ground-truth events via Average Precision.\n    4. Reduction - The multiple AP scores are averaged to produce a single\n    overall score.\n\n    Selection\n\n    With each series there may be a defined set of scoring intervals giving the\n    intervals of time over which zero or more ground-truth events might be\n    annotated in that series. A prediction will be evaluated only if it falls\n    within a scoring interval. These scoring intervals can be chosen to improve\n    the fairness of evaluation by, for instance, ignoring edge-cases or\n    ambiguous events.\n\n    It is recommended that, if used, scoring intervals be provided for training\n    data but not test data.\n\n    Assignment\n\n    For each set of predictions and ground-truths within the same `event x\n    tolerance x series_id` group, we match each ground-truth to the\n    highest-confidence unmatched prediction occurring within the allowed\n    tolerance.\n\n    Some ground-truths may not be matched to a prediction and some predictions\n    may not be matched to a ground-truth. They will still be accounted for in\n    the scoring, however.\n\n    Scoring\n\n    Collecting the events within each `series_id`, we compute an Average\n    Precision score for each `event x tolerance` group. The average precision\n    score is the area under the (step-wise) precision-recall curve generated by\n    decreasing confidence score thresholds over the predictions. In this\n    calculation, matched predictions over the threshold are scored as TP and\n    unmatched predictions as FP. Unmatched ground-truths are scored as FN.\n\n    Reduction\n\n    The final score is the average of the above AP scores, first averaged over\n    tolerance, then over event.\n\n    Parameters\n    ----------\n    solution : pd.DataFrame, with columns:\n\n        `series_id_column_name` identifier for each time series\n\n        `time_column_name` the time of occurence for each event as a numeric type\n\n        `event_column_name` class label for each event\n\n        The solution contains the time of occurence of one or more types of\n        event within one or more time series. The metric expects the solution to\n        contain the same event types as those given in `tolerances`.\n\n        When `use_scoring_intervals == True`, you may include `start` and `end`\n        events to delimit intervals within which detections will be scored.\n        Detected events (from the user submission) outside of these events will\n        be ignored.\n\n    submission : pd.DataFrame, with columns as above and in addition:\n\n        `score_column_name` the predicted confidence score for the detected event\n\n    tolerances : Dict[str, List[float]]\n\n        Maps each event class to a list of timestamp tolerances used\n        for matching detections to ground-truth events.\n\n    use_scoring_intervals: bool, default False\n\n        Whether to ignore predicted events outside intervals delimited\n        by `'start'` and `'end'` events in the solution. When `False`,\n        the solution should not include `'start'` and `'end'` events.\n        See the examples for illustration.\n\n    Returns\n    -------\n    event_detection_ap : float\n        The mean average precision of the detected events.\n\n    Examples\n    --------\n    Detecting `'pass'` events in football:\n    >>> column_names = {\n    ...     'series_id_column_name': 'video_id',\n    ...     'time_column_name': 'time',\n    ...     'event_column_name': 'event',\n    ...     'score_column_name': 'score',\n    ... }\n    >>> tolerances = {'pass': [1.0]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a'],\n    ...     'event': ['pass', 'pass'],\n    ...     'time': [0, 15],\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a'],\n    ...     'event': ['pass', 'pass', 'pass'],\n    ...     'score': [1.0, 0.5, 1.0],\n    ...     'time': [0, 10, 14.5],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names)\n    1.0\n\n    Increasing the confidence score of the false detection above the true\n    detections decreases the AP.\n    >>> submission.loc[1, 'score'] = 1.5\n    >>> score(solution, submission, tolerances, **column_names)\n    0.6666666666666666...\n\n    Likewise, decreasing the confidence score of a true detection below the\n    false detection also decreases the AP.\n    >>> submission.loc[1, 'score'] = 0.5  # reset\n    >>> submission.loc[0, 'score'] = 0.0\n    >>> score(solution, submission, tolerances, **column_names)\n    0.8333333333333333...\n\n    We average AP scores over tolerances. Previously, the detection at 14.5\n    would match, but adding smaller tolerances gives AP scores where it does\n    not match. This results in both a FN, since the ground-truth wasn't\n    detected, and a FP, since the detected event matches no ground-truth.\n    >>> tolerances = {'pass': [0.1, 0.2, 1.0]}\n    >>> score(solution, submission, tolerances, **column_names)\n    0.3888888888888888...\n\n    We also average over time series and over event classes.\n    >>> tolerances = {'pass': [0.5, 1.0], 'challenge': [0.25, 0.50]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'b'],\n    ...     'event': ['pass', 'challenge', 'pass'],\n    ...     'time': [0, 15, 0],  # restart time for new time series b\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'b'],\n    ...     'event': ['pass', 'challenge', 'pass'],\n    ...     'score': [1.0, 0.5, 1.0],\n    ...     'time': [0, 15, 0],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names)\n    1.0\n\n    By adding scoring intervals to the solution, we may choose to ignore\n    detections outside of those intervals.\n    >>> tolerances = {'pass': [1.0]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a', 'a'],\n    ...     'event': ['start', 'pass', 'pass', 'end'],\n    ...     'time': [0, 10, 20, 30],\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a'],\n    ...     'event': ['pass', 'pass', 'pass'],\n    ...     'score': [1.0, 1.0, 1.0],\n    ...     'time': [10, 20, 40],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names, use_scoring_intervals=True)\n    1.0\n\n    \"\"\"\n    solution = solution.to_pandas()\n    submission = submission.to_pandas()\n\n    solution = solution[~solution[time_column_name].isnull()].reset_index(drop=True)\n\n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({\"start\", \"end\"}), (\n        f\"Solution column {event_column_name} must contain the same events \" \"as defined in tolerances.\"\n    )\n    assert pd.api.types.is_numeric_dtype(\n        solution[time_column_name]\n    ), f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{column_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(f\"Submission column '{time_column_name}' must be of numeric type.\")\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(f\"Submission column '{score_column_name}' must be of numeric type.\")\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()[\"series_id_column_name\"] = series_id_column_name\n    globals()[\"time_column_name\"] = time_column_name\n    globals()[\"event_column_name\"] = event_column_name\n    globals()[\"score_column_name\"] = score_column_name\n    globals()[\"use_scoring_intervals\"] = use_scoring_intervals\n    globals()[\"return_detections_matched\"] = return_detections_matched\n\n    return event_detection_ap(solution, submission, tolerances)\n\n\ndef filter_detections(detections: pd.DataFrame, intervals: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Drop detections not inside a scoring interval.\"\"\"\n    detection_time = detections.loc[:, time_column_name].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.\n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)\n\n\ndef find_nearest_time_idx(\n    times: list[int], target_time: int, excluded_indices: set[int], tolerance: float\n) -> tuple[int, int]:\n    \"\"\"Find the index of the nearest time to the target_time that is not in excluded_indices.\n\n    From: https://github.com/tubo213/kaggle-child-mind-institute-detect-sleep-states/blob/main/src/utils/metrics.py\n    \"\"\"\n    idx = bisect_left(times, target_time)\n\n    best_idx = None\n    best_error = float(\"inf\")\n\n    offset_range = min(len(times), tolerance)\n    for offset in range(-offset_range, offset_range):  # Check the exact, one before, and one after\n        check_idx = idx + offset\n        if 0 <= check_idx < len(times) and check_idx not in excluded_indices:\n            error = abs(times[check_idx] - target_time)\n            if error < best_error:\n                best_error = error\n                best_idx = check_idx\n\n    return best_idx, best_error\n\n\ndef match_detections(tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    ground_truths_times = ground_truths.sort_values(time_column_name)[time_column_name].tolist()\n    matched_gt_indices: set[int] = set()\n\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        det_time = getattr(det, time_column_name)\n\n        best_idx, best_error = find_nearest_time_idx(ground_truths_times, det_time, matched_gt_indices, tolerance)\n\n        if best_idx is not None and best_error < tolerance:\n            is_matched[i] = True\n            matched_gt_indices.add(best_idx)\n\n    detections_sorted[\"matched\"] = is_matched\n    return detections_sorted\n\n\ndef precision_recall_curve(\n    matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind=\"stable\")[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\n\ndef event_detection_ap(\n    solution: pd.DataFrame,\n    submission: pd.DataFrame,\n    tolerances: Dict[str, List[float]],\n) -> float:\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution.query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index=\"interval\",\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, [\"start\", \"end\"]]\n            .apply(lambda x: pd.Interval(*x, closed=\"both\"), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = solution.query(\"event not in ['start', 'end']\").reset_index(drop=True)\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched=False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [\n            (ev, tol, vid)\n            for ev in tolerances.keys()\n            for tol in tolerances[ev]\n            for vid in ground_truths[series_id_column_name].unique()\n        ],\n        columns=[event_column_name, \"tolerance\", series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = aggregation_keys.merge(\n        detections_filtered, on=[event_column_name, series_id_column_name], how=\"left\"\n    ).groupby([event_column_name, \"tolerance\", series_id_column_name])\n    ground_truths_grouped = aggregation_keys.merge(\n        ground_truths, on=[event_column_name, series_id_column_name], how=\"left\"\n    ).groupby([event_column_name, \"tolerance\", series_id_column_name])\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(match_detections(dets[\"tolerance\"].iloc[0], gts, dets))\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched.query(\"event in @event_classes\")\n        .groupby([event_column_name, \"tolerance\"])\n        .apply(\n            lambda group: average_precision_score(\n                group[\"matched\"].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    if return_detections_matched:\n        return mean_ap, detections_matched\n\n    return mean_ap","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile src/utils.py\n\nfrom __future__ import annotations\n\nimport functools\nimport gc\nimport time\nfrom contextlib import ContextDecorator\nfrom pathlib import Path\nfrom typing import Any, Callable, Type\n\nimport requests\nimport torch\nfrom google.cloud import storage\nfrom loguru import logger\n\nfrom . import meta_config\n\n\ndef send_line_notification(message: str) -> None:\n    \"\"\"Send notification to LINE.\"\"\"\n    line_token = meta_config.line_token\n    endpoint = \"https://notify-api.line.me/api/notify\"\n    payload = {\"message\": f\"\\n{message}\"}\n    headers = {\"Authorization\": f\"Bearer {line_token}\"}\n    requests.post(endpoint, data=payload, headers=headers)\n\n\nclass line_notification(ContextDecorator):\n    \"\"\"Context-manager for sending notification of finishing execution to LINE.\n    Also functions as a decorator. (Make sure to instantiate with parenthesis.)\n    \"\"\"\n\n    def __enter__(self) -> None:\n        self.start_time = time.time()\n\n    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:\n        if exc_type is None:\n            elapsed_time = (time.time() - self.start_time) / 60\n            send_line_notification(\n                f\"[{meta_config.wandb_project}] {meta_config.run_name} done in {elapsed_time:.1f} min.\"\n            )\n        else:\n            send_line_notification(exc_value)\n\n\ndef freeze(cls) -> Type:\n    \"\"\"Decorator function for fixing class variables.\n\n    Examples\n    --------\n\n        >>> @freeze\n        >>> class config:\n        >>>     x = 10\n        >>>     y = 20\n\n        >>> config.x = 30\n        ValueError: Cannot overwrite config.\n    \"\"\"\n\n    class _Const(type):\n        \"\"\"Metaclass of the configuration class.\n\n        Examples\n        --------\n\n            >>> class config(metaclass=_Const):\n            >>>     x = 10\n            >>>     y = 20\n\n            >>> config.x = 30\n            ValueError: Cannot overwrite config.\n\n        References\n        ----------\n        - https://cream-worker.blog.jp/archives/1077207909.html\n        \"\"\"\n\n        def __setattr__(self, name, value):\n            raise ValueError(\"Cannot overwrite config.\")\n\n    class frozen_cls(cls, metaclass=_Const):\n        pass\n\n    return frozen_cls\n\n\nclass timer(ContextDecorator):\n    \"\"\"Context-manager that logs elapsed time of a process.\n    Also functions as a decorator. (Make sure to instantiate with parenthesis.)\n\n    Paramters\n    ---------\n    message : str\n        The displayed message.\n\n    Examples\n    --------\n    - Usage as a context-manager\n\n        >>> with timer('read csv'):\n        >>>     train_df = pd.read_csv(TRAIN_PATH)\n        [read csv] start.\n        [read csv] done in 0.1 min.\n\n    - Usage as a decorator\n\n        >>> @timer()\n        >>> def read_csv():\n        >>>     train_df = pd.read_csv(TRAIN_PATH)\n        >>>     return train_df\n        >>>\n        >>> train_df = read_csv()\n        [read_csv] start.\n        [read_csv] done in 0.1 min.\n    \"\"\"\n\n    def __init__(self, message: str | None = None) -> None:\n        self.message = message\n\n    def __call__(self, function: Callable) -> Callable:\n        if self.message is None:\n            self.message = function.__name__\n        return super().__call__(function)\n\n    def __enter__(self) -> None:\n        self.start_time = time.time()\n        logger.opt(colors=True).info(f\"<green>[{self.message}]</green> start.\")\n\n    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:\n        if exc_type is None:\n            elapsed_time = time.time() - self.start_time\n            logger.opt(colors=True).info(f\"<green>[{self.message}]</green> done in {elapsed_time / 60:.1f} min.\")\n\n\ndef check_gcs_file_exists(gcs_path: str) -> bool:\n    \"\"\"Check if the file exists in the specified path on the GCS.\n\n    Parameters\n    ----------\n    gcs_path : str\n        File path on the GCS.\n\n    Returns\n    -------\n    bool\n        Whether the path points to an existing file.\n    \"\"\"\n    client = storage.Client(meta_config.gcp_project)\n    bucket = client.get_bucket(meta_config.gcp_bucket)\n    exists = any(bucket.list_blobs(prefix=gcs_path))\n    return exists\n\n\n@timer()\ndef download_from_gcs(path: str, gcs_prefix: str | None = None, keep_dir: bool = False) -> None:\n    \"\"\"Download a directory from Google Cloud Storage.\n\n    Parameters\n    ----------\n    path : str\n        The local path to the directory to download.\n    gcs_prefix : str\n        The GCS path of the directory you want to download.\n    keep_dir : bool\n        Whether to keep the GCS directory structure.\n    \"\"\"\n    client = storage.Client(meta_config.gcp_project)\n    bucket = client.get_bucket(meta_config.gcp_bucket)\n    gcs_dir = Path(gcs_prefix).parent\n\n    for blob in bucket.list_blobs(prefix=gcs_prefix):\n        name = blob.name if keep_dir else Path(blob.name).relative_to(gcs_dir)\n        download_path = Path(path) / name\n        download_path.parent.mkdir(parents=True, exist_ok=True)\n        blob.download_to_filename(filename=str(download_path))\n\n\n@timer()\ndef upload_to_gcs(path: str, gcs_prefix: str = \"\") -> None:\n    \"\"\"Upload a directory to Google Cloud Storage.\n\n    Parameters\n    ----------\n    path : str\n        The local path to the directory you want to upload.\n    gcs_prefix : str\n        The GCS path to the directory to upload.\n    \"\"\"\n    if gcs_prefix != \"\" and not gcs_prefix.endswith(\"/\"):\n        gcs_prefix += \"/\"\n\n    def _upload_to_gcs(bucket: storage.bucket.Bucket, path: Path, dir_name: str = \"\"):\n        if path.is_file():\n            file_name = dir_name + path.name\n            blob = bucket.blob(file_name)\n            blob.upload_from_filename(filename=str(path))\n        else:\n            for child_path in path.iterdir():\n                _upload_to_gcs(bucket, child_path, dir_name + path.name + \"/\")\n\n    client = storage.Client(meta_config.gcp_project)\n    bucket = client.get_bucket(meta_config.gcp_bucket)\n    _upload_to_gcs(bucket, Path(path), dir_name=gcs_prefix)\n\n\ndef clear_memory(function: Callable) -> Callable:\n    \"\"\"Decorator function for clearing memory cache.\"\"\"\n\n    @functools.wraps(function)\n    def _clear_memory(*args, **kwargs):\n        function(*args, **kwargs)\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    return _clear_memory","metadata":{},"execution_count":null,"outputs":[]}]}