{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# How Event Detection AP Works\n\n-   The Event Detection AP is straight forward, but it has many objects in it.\n-   Here, I executed the code line by line with example data.\n-   You may get some hint for making the best submission for this competetition!","metadata":{}},{"cell_type":"markdown","source":"# Setup","metadata":{}},{"cell_type":"markdown","source":"## Import","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom typing import Dict, List, Tuple\n\nfrom matplotlib import pyplot as plt\n\nfrom metric import score as metric_score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-25T12:55:23.955260Z","iopub.execute_input":"2023-09-25T12:55:23.955793Z","iopub.status.idle":"2023-09-25T12:55:23.963659Z","shell.execute_reply.started":"2023-09-25T12:55:23.955752Z","shell.execute_reply":"2023-09-25T12:55:23.961974Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Common Parameters","metadata":{}},{"cell_type":"code","source":"tolerances = {\n    \"onset\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    \"wakeup\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n}\n\nseries_id_column_name = 'series_idx'\ntime_column_name = 'step'\nevent_column_name = 'event'\nscore_column_name = 'score'\n\ncommon_pars = {\n    \"tolerances\": tolerances,\n    \"series_id_column_name\": series_id_column_name,\n    \"time_column_name\": time_column_name,\n    \"event_column_name\": event_column_name,\n    \"score_column_name\": score_column_name,\n}\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T11:54:08.192531Z","iopub.execute_input":"2023-09-25T11:54:08.193072Z","iopub.status.idle":"2023-09-25T11:54:08.207946Z","shell.execute_reply.started":"2023-09-25T11:54:08.193030Z","shell.execute_reply":"2023-09-25T11:54:08.206304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pseudo Data\n\n-   solution has the ground_truth value.\n-   submission has example predictions.\n    -   First 6 rows were correctly predicted, while the last 2 rows were completely missed.","metadata":{}},{"cell_type":"code","source":"# solution\nsol_data = {\n    \"series_idx\": np.array([1, 1, 2, 2, 2, 2]),\n    \"event\": np.array([\"onset\", \"wakeup\", \"onset\", \"wakeup\", \"onset\", \"wakeup\"]),\n    \"step\": np.array([1000, 1500, 1100, 1820, 5010, 8800]),\n}\n\nsolution = pd.DataFrame(sol_data)\n\n# submission\nsub_data = {\n    \"series_idx\": np.array([1, 1, 2, 2, 2, 2, 2, 2]),\n    \"event\": np.array([\"onset\", \"wakeup\", \"onset\", \"wakeup\", \"onset\", \"wakeup\", \"onset\", \"wakeup\"]),\n    \"step\": np.array([950, 1500, 1200, 2000, 4600, 8800, 12080, 14000]),\n    \"score\": np.array([0.9, 0.9, 0.8, 0.8, 0.7, 0.7, 0.1, 0.1]),\n}\n\nsubmission = pd.DataFrame(sub_data)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T11:55:50.360860Z","iopub.execute_input":"2023-09-25T11:55:50.361334Z","iopub.status.idle":"2023-09-25T11:55:50.375157Z","shell.execute_reply.started":"2023-09-25T11:55:50.361298Z","shell.execute_reply":"2023-09-25T11:55:50.373150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution","metadata":{"execution":{"iopub.status.busy":"2023-09-25T11:55:51.004146Z","iopub.execute_input":"2023-09-25T11:55:51.004786Z","iopub.status.idle":"2023-09-25T11:55:51.019593Z","shell.execute_reply.started":"2023-09-25T11:55:51.004739Z","shell.execute_reply":"2023-09-25T11:55:51.017875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-09-25T11:55:51.605919Z","iopub.execute_input":"2023-09-25T11:55:51.606769Z","iopub.status.idle":"2023-09-25T11:55:51.621425Z","shell.execute_reply.started":"2023-09-25T11:55:51.606721Z","shell.execute_reply":"2023-09-25T11:55:51.619653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation\n\n-   The return of the AP function was 0.577.\n-   From now on, let's reproduce the metric score above, by executing the function line-by-line.\n","metadata":{}},{"cell_type":"code","source":"print(\"metric_score: \", metric_score(solution, submission, **common_pars))","metadata":{"execution":{"iopub.status.busy":"2023-09-25T11:56:08.076172Z","iopub.execute_input":"2023-09-25T11:56:08.076618Z","iopub.status.idle":"2023-09-25T11:56:08.292005Z","shell.execute_reply.started":"2023-09-25T11:56:08.076585Z","shell.execute_reply":"2023-09-25T11:56:08.290459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reproduction Step 1: `score` function [L26](https://www.kaggle.com/code/metric/event-detection-ap#kln-26)\n\n```python\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n) -> float:\n    # There\n    # Are\n    # So\n    # Many\n    # Robustness\n    # Checks...\n    \n    # And finally,\n    return event_detection_ap(solution, submission, tolerances)\n```\n\n-   The `socre` function just calls the `event_detection_ap` function.","metadata":{}},{"cell_type":"markdown","source":"## Reproduction Step 2: `event_detection_ap` [L345](https://www.kaggle.com/code/metric/event-detection-ap#kln-345)\n\n-   Part 1: Extract ground-truth events.\n-   Part 2: Map each event class to its prevalence (needed for recall calculation)\n-   Part 3: Create table for detections with a column indicating a match to a ground-truth event\n-   Part 4: Create tables\n-   Part 5: Match detections to ground truth events by evaluation group\n-   Part 6: Compute AP per event x tolerance group\n-   Part 7: Average over tolerances, then over event classes\n\n```python\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n    # Extract scoring intervals.\n    # ... skipped as we are not using this option.\n\n    # Part 1: Extract ground-truth events.\n    ground_truths = (solution.query(\"event not in ['start', 'end']\").reset_index(drop=True))\n\n    # Part 2: Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n    \n    # Part 3: Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n    # ... skipped again. we are not using scoring intervals.\n    detections_filtered = detections\n    \n    # Part 4: Create tables\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    \n    # Part 5: Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Part 6: Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    \n    # Part 7: Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap\n```\n","metadata":{}},{"cell_type":"markdown","source":"## Part 1: Extract ground-truth events\n\n-   This part does nothing. Just assign `solution` into `ground_truths`.\n-   There's no 'start' or 'end' in ours.","metadata":{}},{"cell_type":"code","source":"ground_truths = (\n    solution\n    .query(\"event not in ['start', 'end']\")\n    .reset_index(drop=True)\n)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:05:20.736891Z","iopub.execute_input":"2023-09-25T12:05:20.737429Z","iopub.status.idle":"2023-09-25T12:05:20.748258Z","shell.execute_reply.started":"2023-09-25T12:05:20.737389Z","shell.execute_reply":"2023-09-25T12:05:20.746882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truths","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:05:20.966847Z","iopub.execute_input":"2023-09-25T12:05:20.967291Z","iopub.status.idle":"2023-09-25T12:05:20.978280Z","shell.execute_reply.started":"2023-09-25T12:05:20.967229Z","shell.execute_reply":"2023-09-25T12:05:20.977391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 2: Map each event class to its prevalence (needed for recall calculation)\n","metadata":{}},{"cell_type":"code","source":"class_counts = ground_truths.value_counts(event_column_name).to_dict()\nprint(class_counts)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:08:23.408169Z","iopub.execute_input":"2023-09-25T12:08:23.408622Z","iopub.status.idle":"2023-09-25T12:08:23.418528Z","shell.execute_reply.started":"2023-09-25T12:08:23.408587Z","shell.execute_reply":"2023-09-25T12:08:23.416513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 3: Create table for detections with a column indicating a match to a ground-truth event\n\n-   Create `detections` and `detections_filtered`, which are equal to `submission` with a new column `matched`.","metadata":{}},{"cell_type":"code","source":"detections = submission.assign(matched = False)\ndetections_filtered = detections\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:09:24.938436Z","iopub.execute_input":"2023-09-25T12:09:24.938865Z","iopub.status.idle":"2023-09-25T12:09:24.945346Z","shell.execute_reply.started":"2023-09-25T12:09:24.938831Z","shell.execute_reply":"2023-09-25T12:09:24.944279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detections_filtered","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:09:26.652608Z","iopub.execute_input":"2023-09-25T12:09:26.653210Z","iopub.status.idle":"2023-09-25T12:09:26.673594Z","shell.execute_reply.started":"2023-09-25T12:09:26.653164Z","shell.execute_reply":"2023-09-25T12:09:26.671474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 4: Create tables\n\n-   Create tables that will store the match success-or-fail within the tolerance tested.\n-   `aggregation_keys`: for each event, series_idx and the tolerance, the matching will be tried.\n-   `detections_grouped`: the submission grouped by the key (in aggregation_keys)\n-   `ground_truths_grouped`: the solution grouped by the key (in aggregation_keys)\n-   These two grouped data frames, `detections_grouped` and `ground_truths_grouped` will be matched!","metadata":{}},{"cell_type":"code","source":"aggregation_keys = pd.DataFrame(\n    [(ev, tol, vid)\n     for ev in tolerances.keys()\n     for tol in tolerances[ev]\n     for vid in ground_truths[series_id_column_name].unique()],\n    columns=[event_column_name, 'tolerance', series_id_column_name],\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:11:30.829420Z","iopub.execute_input":"2023-09-25T12:11:30.829900Z","iopub.status.idle":"2023-09-25T12:11:30.855832Z","shell.execute_reply.started":"2023-09-25T12:11:30.829861Z","shell.execute_reply":"2023-09-25T12:11:30.854367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregation_keys","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:11:34.341667Z","iopub.execute_input":"2023-09-25T12:11:34.342177Z","iopub.status.idle":"2023-09-25T12:11:34.361675Z","shell.execute_reply.started":"2023-09-25T12:11:34.342136Z","shell.execute_reply":"2023-09-25T12:11:34.360341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregation_keys.merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:16:48.502785Z","iopub.execute_input":"2023-09-25T12:16:48.503309Z","iopub.status.idle":"2023-09-25T12:16:48.532082Z","shell.execute_reply.started":"2023-09-25T12:16:48.503249Z","shell.execute_reply":"2023-09-25T12:16:48.529848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detections_grouped = (\n    aggregation_keys\n    .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n    .groupby([event_column_name, 'tolerance', series_id_column_name])\n)\n\ndetections_grouped.groups","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:16:58.792030Z","iopub.execute_input":"2023-09-25T12:16:58.792594Z","iopub.status.idle":"2023-09-25T12:16:58.826718Z","shell.execute_reply.started":"2023-09-25T12:16:58.792557Z","shell.execute_reply":"2023-09-25T12:16:58.825201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"aggregation_keys.merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:17:10.889933Z","iopub.execute_input":"2023-09-25T12:17:10.890449Z","iopub.status.idle":"2023-09-25T12:17:10.919955Z","shell.execute_reply.started":"2023-09-25T12:17:10.890409Z","shell.execute_reply":"2023-09-25T12:17:10.918349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ground_truths_grouped = (\n    aggregation_keys\n    .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n    .groupby([event_column_name, 'tolerance', series_id_column_name])\n)\nground_truths_grouped.groups\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:17:20.199377Z","iopub.execute_input":"2023-09-25T12:17:20.199914Z","iopub.status.idle":"2023-09-25T12:17:20.238402Z","shell.execute_reply.started":"2023-09-25T12:17:20.199872Z","shell.execute_reply":"2023-09-25T12:17:20.237077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 5: Match detections to ground truth events by evaluation group\n\n-   Here the function `match_detections` called! Let's see the match function!\n\n```python\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n```\n","metadata":{}},{"cell_type":"markdown","source":"# Reproduction Step 3: `match_detections` [L282](https://www.kaggle.com/code/metric/event-detection-ap#kln-282)\n\n-   Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\n\n```python\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n```","metadata":{}},{"cell_type":"markdown","source":"## Example run of `match_detections` function","metadata":{}},{"cell_type":"code","source":"# For example, let's try with the first group key.\nkey = next(aggregation_keys.itertuples(index=False))\nprint(\"key: \", key)\nprint(\"     It means that we are evaluating \", key.series_idx, \" ID.\")\nprint(\"     Looking for \", key.event, \" event within\")\nprint(\"     the tolerance of \", key.tolerance)\n\ndets = detections_grouped.get_group(key)\ngts = ground_truths_grouped.get_group(key)\n\n# match_detections(dets['tolerance'].iloc[0], gts, dets)\nMD_tolerance = dets['tolerance'].iloc[0]\nMD_ground_truths = gts\nMD_detections = dets\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:30:44.427222Z","iopub.execute_input":"2023-09-25T12:30:44.427738Z","iopub.status.idle":"2023-09-25T12:30:44.437936Z","shell.execute_reply.started":"2023-09-25T12:30:44.427699Z","shell.execute_reply":"2023-09-25T12:30:44.436883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sort submission\ndetections_sorted = MD_detections.sort_values(score_column_name, ascending=False).dropna()\n\n# init match\nis_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\ngts_matched = set()\n\n# for each submissions, find best matching ground-truth solutions\n# within the tolerance.\n# If there's no ground-truth within the tolerance, the `gts_matched`\n# is checked as False.\nfor i, det in enumerate(detections_sorted.itertuples(index=False)):\n    best_error = MD_tolerance\n    best_gt = None\n\n    # if a ground-truth row was found within the tolerance\n    for gt in MD_ground_truths.itertuples(index=False):\n        error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n        if error < best_error and gt not in gts_matched:\n            best_gt = gt\n            best_error = error\n\n    if best_gt is not None:\n        is_matched[i] = True\n        gts_matched.add(best_gt)\n\ndetections_sorted['matched'] = is_matched\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:32:44.101674Z","iopub.execute_input":"2023-09-25T12:32:44.102058Z","iopub.status.idle":"2023-09-25T12:32:44.115641Z","shell.execute_reply.started":"2023-09-25T12:32:44.102027Z","shell.execute_reply":"2023-09-25T12:32:44.114376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This example failed to match, as the tolerance was too small.","metadata":{}},{"cell_type":"code","source":"detections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:32:45.277077Z","iopub.execute_input":"2023-09-25T12:32:45.277517Z","iopub.status.idle":"2023-09-25T12:32:45.291140Z","shell.execute_reply.started":"2023-09-25T12:32:45.277486Z","shell.execute_reply":"2023-09-25T12:32:45.289800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Another example run\n\n-   Larger tolerance made it matching.","metadata":{}},{"cell_type":"code","source":"# For example, let's try with the first group key.\nkey_iter = aggregation_keys.itertuples(index=False)\n_ = [next(key_iter) for _ in range(10)]\nkey = next(key_iter)\nprint(\"key: \", key)\nprint(\"     It means that we are evaluating \", key.series_idx, \" ID.\")\nprint(\"     Looking for \", key.event, \" event within\")\nprint(\"     the tolerance of \", key.tolerance)\n\ndets = detections_grouped.get_group(key)\ngts = ground_truths_grouped.get_group(key)\n\n# match_detections(dets['tolerance'].iloc[0], gts, dets)\nMD_tolerance = dets['tolerance'].iloc[0]\nMD_ground_truths = gts\nMD_detections = dets\n\n# sort submission\ndetections_sorted = MD_detections.sort_values(score_column_name, ascending=False).dropna()\n\n# init match\nis_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\ngts_matched = set()\n\n# for each submissions, find best matching ground-truth solutions\n# within the tolerance.\n# If there's no ground-truth within the tolerance, the `gts_matched`\n# is checked as False.\nfor i, det in enumerate(detections_sorted.itertuples(index=False)):\n    best_error = MD_tolerance\n    best_gt = None\n\n    # if a ground-truth row was found within the tolerance\n    for gt in MD_ground_truths.itertuples(index=False):\n        error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n        if error < best_error and gt not in gts_matched:\n            best_gt = gt\n            best_error = error\n\n    if best_gt is not None:\n        is_matched[i] = True\n        gts_matched.add(best_gt)\n\ndetections_sorted['matched'] = is_matched\n\ndetections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:35:18.996943Z","iopub.execute_input":"2023-09-25T12:35:18.997392Z","iopub.status.idle":"2023-09-25T12:35:19.029681Z","shell.execute_reply.started":"2023-09-25T12:35:18.997358Z","shell.execute_reply":"2023-09-25T12:35:19.027722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define the function `match_detections`","metadata":{}},{"cell_type":"code","source":"def match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:37:57.134490Z","iopub.execute_input":"2023-09-25T12:37:57.135079Z","iopub.status.idle":"2023-09-25T12:37:57.146983Z","shell.execute_reply.started":"2023-09-25T12:37:57.135038Z","shell.execute_reply":"2023-09-25T12:37:57.145133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reproduction Step 2: `event_detection_ap` [L345](https://www.kaggle.com/code/metric/event-detection-ap#kln-345). Resume from Part 5\n\n## Part 5: Match detections to ground truth events by evaluation group\n\n-   The resulting DataFrame `detections_matched` means if the prediction was matched with ground-truth within the tolerance given.","metadata":{}},{"cell_type":"code","source":"detections_matched = []\nfor key in aggregation_keys.itertuples(index=False):\n    dets = detections_grouped.get_group(key)\n    gts = ground_truths_grouped.get_group(key)\n    detections_matched.append(\n        match_detections(dets['tolerance'].iloc[0], gts, dets)\n    )\ndetections_matched = pd.concat(detections_matched)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:38:03.609124Z","iopub.execute_input":"2023-09-25T12:38:03.609599Z","iopub.status.idle":"2023-09-25T12:38:04.361841Z","shell.execute_reply.started":"2023-09-25T12:38:03.609564Z","shell.execute_reply":"2023-09-25T12:38:04.360192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"detections_matched","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:38:05.908099Z","iopub.execute_input":"2023-09-25T12:38:05.908539Z","iopub.status.idle":"2023-09-25T12:38:05.932440Z","shell.execute_reply.started":"2023-09-25T12:38:05.908486Z","shell.execute_reply":"2023-09-25T12:38:05.931388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 6: Compute AP per event x tolerance group\n\n-   This is the evaluation part. We call `average_precision_score` function.\n-   To understand this, we should understand the `precision_recall_curve` function first.","metadata":{}},{"cell_type":"markdown","source":"# Reproduction Step 3: Precision-Recal Score [L339](https://www.kaggle.com/code/metric/event-detection-ap#kln-339)","metadata":{}},{"cell_type":"markdown","source":"## Example Precision-Recall Curve","metadata":{}},{"cell_type":"code","source":"def precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:54:20.347573Z","iopub.execute_input":"2023-09-25T12:54:20.348125Z","iopub.status.idle":"2023-09-25T12:54:20.361688Z","shell.execute_reply.started":"2023-09-25T12:54:20.348085Z","shell.execute_reply":"2023-09-25T12:54:20.359755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### An example data for drawing the curve","metadata":{}},{"cell_type":"code","source":"# example data for the precision-recall curve\npseudo_group = detections_matched[(detections_matched[\"event\"] == \"onset\") & (detections_matched[\"tolerance\"] == 180)]\n\npseudo_matches = pseudo_group[\"matched\"].to_numpy()\npseudo_scores = pseudo_group[\"score\"].to_numpy()\npseudo_p = class_counts[pseudo_group[\"event\"].iat[0]]","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:16.640529Z","iopub.execute_input":"2023-09-25T12:59:16.641015Z","iopub.status.idle":"2023-09-25T12:59:16.651448Z","shell.execute_reply.started":"2023-09-25T12:59:16.640967Z","shell.execute_reply":"2023-09-25T12:59:16.649568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_group","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:16.845716Z","iopub.execute_input":"2023-09-25T12:59:16.846204Z","iopub.status.idle":"2023-09-25T12:59:16.863882Z","shell.execute_reply.started":"2023-09-25T12:59:16.846164Z","shell.execute_reply":"2023-09-25T12:59:16.862094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_matches","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:17.316260Z","iopub.execute_input":"2023-09-25T12:59:17.317481Z","iopub.status.idle":"2023-09-25T12:59:17.330741Z","shell.execute_reply.started":"2023-09-25T12:59:17.317405Z","shell.execute_reply":"2023-09-25T12:59:17.329152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_scores","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:17.849930Z","iopub.execute_input":"2023-09-25T12:59:17.850521Z","iopub.status.idle":"2023-09-25T12:59:17.861607Z","shell.execute_reply.started":"2023-09-25T12:59:17.850475Z","shell.execute_reply":"2023-09-25T12:59:17.859825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pseudo_p","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:18.216994Z","iopub.execute_input":"2023-09-25T12:59:18.217655Z","iopub.status.idle":"2023-09-25T12:59:18.229728Z","shell.execute_reply.started":"2023-09-25T12:59:18.217560Z","shell.execute_reply":"2023-09-25T12:59:18.226385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate the precision-recall curve","metadata":{}},{"cell_type":"code","source":"pseudo_precision, pseudo_recall, _ = precision_recall_curve(pseudo_matches, pseudo_scores, pseudo_p)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:19.305976Z","iopub.execute_input":"2023-09-25T12:59:19.306462Z","iopub.status.idle":"2023-09-25T12:59:19.312670Z","shell.execute_reply.started":"2023-09-25T12:59:19.306428Z","shell.execute_reply":"2023-09-25T12:59:19.311445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(pseudo_scores, pseudo_matches)\n\nplt.xlabel(\"Scores\")\nplt.ylabel(\"Matches\")\nplt.xlim(0, 1.05)\nplt.ylim(0, 1.05)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:02:51.568107Z","iopub.execute_input":"2023-09-25T13:02:51.568586Z","iopub.status.idle":"2023-09-25T13:02:51.783761Z","shell.execute_reply.started":"2023-09-25T13:02:51.568545Z","shell.execute_reply":"2023-09-25T13:02:51.782314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.fill_between(pseudo_recall, 0., pseudo_precision)\nplt.scatter(pseudo_recall, pseudo_precision)\n\nplt.xlabel(\"Recall\")\nplt.ylabel(\"Precision\")\nplt.xlim(0, 1.05)\nplt.ylim(0, 1.05)\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:00:18.630323Z","iopub.execute_input":"2023-09-25T13:00:18.630713Z","iopub.status.idle":"2023-09-25T13:00:18.912084Z","shell.execute_reply.started":"2023-09-25T13:00:18.630682Z","shell.execute_reply":"2023-09-25T13:00:18.910814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:25.137445Z","iopub.execute_input":"2023-09-25T12:59:25.137869Z","iopub.status.idle":"2023-09-25T12:59:25.146471Z","shell.execute_reply.started":"2023-09-25T12:59:25.137836Z","shell.execute_reply":"2023-09-25T12:59:25.144456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"average_precision_score(pseudo_matches, pseudo_scores, pseudo_p)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T12:59:25.307577Z","iopub.execute_input":"2023-09-25T12:59:25.308633Z","iopub.status.idle":"2023-09-25T12:59:25.319812Z","shell.execute_reply.started":"2023-09-25T12:59:25.308570Z","shell.execute_reply":"2023-09-25T12:59:25.318114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reproduction Step 2: `event_detection_ap` [L345](https://www.kaggle.com/code/metric/event-detection-ap#kln-345). Resume from Part 6\n\n## Part 6: Compute AP per event x tolerance group\n\n-   For each events and tolerances, evaluate the average precision score.","metadata":{}},{"cell_type":"code","source":"event_classes = ground_truths[event_column_name].unique()\nprint(\"event_classes\")\nprint(event_classes)\n\nap_table = (\n    detections_matched\n    .query(\"event in @event_classes\")  # Robustness check\n    .groupby([event_column_name, 'tolerance']).apply(\n        lambda group: average_precision_score(\n            group['matched'].to_numpy(),\n            group[score_column_name].to_numpy(),\n            class_counts[group[event_column_name].iat[0]],\n        )\n    )\n)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:05:41.057171Z","iopub.execute_input":"2023-09-25T13:05:41.057638Z","iopub.status.idle":"2023-09-25T13:05:41.083364Z","shell.execute_reply.started":"2023-09-25T13:05:41.057604Z","shell.execute_reply":"2023-09-25T13:05:41.081578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ap_table","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:05:42.729216Z","iopub.execute_input":"2023-09-25T13:05:42.730494Z","iopub.status.idle":"2023-09-25T13:05:42.743112Z","shell.execute_reply.started":"2023-09-25T13:05:42.730451Z","shell.execute_reply":"2023-09-25T13:05:42.741365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Part 7: Average over tolerances, then over event classes\n\n-   Make the average over all the tolerances and events.\n","metadata":{}},{"cell_type":"code","source":"mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:07:10.959128Z","iopub.execute_input":"2023-09-25T13:07:10.959795Z","iopub.status.idle":"2023-09-25T13:07:10.969973Z","shell.execute_reply.started":"2023-09-25T13:07:10.959746Z","shell.execute_reply":"2023-09-25T13:07:10.968362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Conclusion\n\n-   The line-by-line result matches with the function return value.\n-   You can play with this code for dig more into the metric function.","metadata":{}},{"cell_type":"code","source":"print(\"         Line-by-line result: \", mean_ap)\nprint(\"metric_score function return: \", metric_score(solution, submission, **common_pars))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-25T13:08:02.121884Z","iopub.execute_input":"2023-09-25T13:08:02.122413Z","iopub.status.idle":"2023-09-25T13:08:02.341200Z","shell.execute_reply.started":"2023-09-25T13:08:02.122374Z","shell.execute_reply":"2023-09-25T13:08:02.339723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## END-OF-NOTE","metadata":{}}]}