{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Playing with the [Child Mind Institute - Detect Sleep States](https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states) competition metric\n\nThe source of the meric is in the notebook [\"Event Detection AP\"](https://www.kaggle.com/code/metric/event-detection-ap/notebook).\n\nThe aim of this notebook is to perform experiments so as to help to become more familiar with the scoring process.\n\nThe undelying metric for each individual series is the **area under the precision-recall curve**.\n\n# Notes:\n\n* In this competition I think **all** of the data is scored, so the `filter_detections` function has been excluded.\n\n* There is no reference to `row_id` so I am not sure if this column is actually ever used?\n\n* 1 step = 5 seconds\n* 1 minute  = 12 steps\n","metadata":{}},{"cell_type":"markdown","source":"# The source code","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport pandas.api.types\nfrom typing import Dict, List, Tuple\n\n# tolerances in steps\ntolerances = {\n    \"onset\":  [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    \"wakeup\": [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n}","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-16T17:09:20.018096Z","iopub.execute_input":"2023-09-16T17:09:20.018435Z","iopub.status.idle":"2023-09-16T17:09:20.442167Z","shell.execute_reply.started":"2023-09-16T17:09:20.018403Z","shell.execute_reply":"2023-09-16T17:09:20.441211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id_column_name = \"series_id\"\ntime_column_name = \"step\"\nevent_column_name = \"event\"\nscore_column_name = \"score\"\nuse_scoring_intervals = None","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.443799Z","iopub.execute_input":"2023-09-16T17:09:20.444639Z","iopub.status.idle":"2023-09-16T17:09:20.450820Z","shell.execute_reply.started":"2023-09-16T17:09:20.444595Z","shell.execute_reply":"2023-09-16T17:09:20.449661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n) -> float:\n    \n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{target_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.453372Z","iopub.execute_input":"2023-09-16T17:09:20.453679Z","iopub.status.idle":"2023-09-16T17:09:20.466001Z","shell.execute_reply.started":"2023-09-16T17:09:20.453652Z","shell.execute_reply":"2023-09-16T17:09:20.464767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.467783Z","iopub.execute_input":"2023-09-16T17:09:20.468494Z","iopub.status.idle":"2023-09-16T17:09:20.492440Z","shell.execute_reply.started":"2023-09-16T17:09:20.468453Z","shell.execute_reply":"2023-09-16T17:09:20.491438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.493812Z","iopub.execute_input":"2023-09-16T17:09:20.494252Z","iopub.status.idle":"2023-09-16T17:09:20.508968Z","shell.execute_reply.started":"2023-09-16T17:09:20.494220Z","shell.execute_reply":"2023-09-16T17:09:20.508069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.510577Z","iopub.execute_input":"2023-09-16T17:09:20.511217Z","iopub.status.idle":"2023-09-16T17:09:20.525479Z","shell.execute_reply.started":"2023-09-16T17:09:20.511175Z","shell.execute_reply":"2023-09-16T17:09:20.524299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.527046Z","iopub.execute_input":"2023-09-16T17:09:20.527472Z","iopub.status.idle":"2023-09-16T17:09:20.541687Z","shell.execute_reply.started":"2023-09-16T17:09:20.527430Z","shell.execute_reply":"2023-09-16T17:09:20.540798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test: Calculate the **Average Precision** (\"AP\") for one series\nEach series AP score is averaged to produce the final LB score.","metadata":{}},{"cell_type":"code","source":"train_events = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\n\n# extract just one series to be the \"ground truth\"\nsolution =  train_events.loc[(train_events[\"series_id\"] == \"d515236bdeec\")].copy()\n\n# create a perfect submission\nperfect_submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\n\n# create a perfect score column\nperfect_submission[\"score\"] = 1","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.545791Z","iopub.execute_input":"2023-09-16T17:09:20.546507Z","iopub.status.idle":"2023-09-16T17:09:20.616119Z","shell.execute_reply.started":"2023-09-16T17:09:20.546466Z","shell.execute_reply":"2023-09-16T17:09:20.615115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# take a look\nperfect_submission","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.617526Z","iopub.execute_input":"2023-09-16T17:09:20.618066Z","iopub.status.idle":"2023-09-16T17:09:20.640214Z","shell.execute_reply.started":"2023-09-16T17:09:20.618034Z","shell.execute_reply":"2023-09-16T17:09:20.639297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"calculate the score","metadata":{}},{"cell_type":"code","source":"event_detection_ap(solution, perfect_submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.641448Z","iopub.execute_input":"2023-09-16T17:09:20.641967Z","iopub.status.idle":"2023-09-16T17:09:20.934627Z","shell.execute_reply.started":"2023-09-16T17:09:20.641935Z","shell.execute_reply":"2023-09-16T17:09:20.933592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Indeed a perfect score\n\n# Setting a different global `score` value","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 0.5\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:20.936033Z","iopub.execute_input":"2023-09-16T17:09:20.936450Z","iopub.status.idle":"2023-09-16T17:09:21.287600Z","shell.execute_reply.started":"2023-09-16T17:09:20.936409Z","shell.execute_reply":"2023-09-16T17:09:21.286260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 0.1\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:21.289204Z","iopub.execute_input":"2023-09-16T17:09:21.289918Z","iopub.status.idle":"2023-09-16T17:09:21.641159Z","shell.execute_reply.started":"2023-09-16T17:09:21.289862Z","shell.execute_reply":"2023-09-16T17:09:21.639928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If all of the `score` values are the same,  the `score` value seems to make no difference","metadata":{}},{"cell_type":"markdown","source":"# Zero  score (non-event?)","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 0\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:21.642904Z","iopub.execute_input":"2023-09-16T17:09:21.643578Z","iopub.status.idle":"2023-09-16T17:09:21.939006Z","shell.execute_reply.started":"2023-09-16T17:09:21.643534Z","shell.execute_reply":"2023-09-16T17:09:21.937813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We still have a result of `1`\n\n# Shifting the `step`\n\nLess than 1 minute off (less than 12 steps)","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"step\"] = submission[\"step\"]+10\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:21.940420Z","iopub.execute_input":"2023-09-16T17:09:21.941328Z","iopub.status.idle":"2023-09-16T17:09:22.331192Z","shell.execute_reply.started":"2023-09-16T17:09:21.941290Z","shell.execute_reply":"2023-09-16T17:09:22.327307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"just over 1 minute off:","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"step\"] = submission[\"step\"]+15\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:22.332822Z","iopub.execute_input":"2023-09-16T17:09:22.333505Z","iopub.status.idle":"2023-09-16T17:09:22.685059Z","shell.execute_reply.started":"2023-09-16T17:09:22.333465Z","shell.execute_reply":"2023-09-16T17:09:22.683658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"step\"] = submission[\"step\"]+350\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:22.687205Z","iopub.execute_input":"2023-09-16T17:09:22.688093Z","iopub.status.idle":"2023-09-16T17:09:22.989264Z","shell.execute_reply.started":"2023-09-16T17:09:22.688039Z","shell.execute_reply":"2023-09-16T17:09:22.988483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"If our `event` is more than 30 minutes from the `ground truth`  event the score is zero","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"step\"] = submission[\"step\"]+370\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:22.990497Z","iopub.execute_input":"2023-09-16T17:09:22.990977Z","iopub.status.idle":"2023-09-16T17:09:23.275292Z","shell.execute_reply.started":"2023-09-16T17:09:22.990946Z","shell.execute_reply":"2023-09-16T17:09:23.274084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# change both `step` and `score`","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 0.1\nsubmission[\"step\"] = submission[\"step\"]+350\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:23.276647Z","iopub.execute_input":"2023-09-16T17:09:23.277272Z","iopub.status.idle":"2023-09-16T17:09:23.556661Z","shell.execute_reply.started":"2023-09-16T17:09:23.277233Z","shell.execute_reply":"2023-09-16T17:09:23.555423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random `score` with no time shift","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = np.random.uniform(size=len(submission))\nsubmission[\"step\"] = submission[\"step\"]\n\n# take a look\nsubmission.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:23.558509Z","iopub.execute_input":"2023-09-16T17:09:23.558935Z","iopub.status.idle":"2023-09-16T17:09:23.575476Z","shell.execute_reply.started":"2023-09-16T17:09:23.558896Z","shell.execute_reply":"2023-09-16T17:09:23.574764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:23.576594Z","iopub.execute_input":"2023-09-16T17:09:23.577059Z","iopub.status.idle":"2023-09-16T17:09:23.862172Z","shell.execute_reply.started":"2023-09-16T17:09:23.577031Z","shell.execute_reply":"2023-09-16T17:09:23.860781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Only `wakeup` events","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"event\"] = \"wakeup\"\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:23.863900Z","iopub.execute_input":"2023-09-16T17:09:23.864352Z","iopub.status.idle":"2023-09-16T17:09:24.205229Z","shell.execute_reply.started":"2023-09-16T17:09:23.864306Z","shell.execute_reply":"2023-09-16T17:09:24.203987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Only `onset` events","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission[\"event\"] = \"onset\"\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:24.206871Z","iopub.execute_input":"2023-09-16T17:09:24.207270Z","iopub.status.idle":"2023-09-16T17:09:24.552180Z","shell.execute_reply.started":"2023-09-16T17:09:24.207232Z","shell.execute_reply":"2023-09-16T17:09:24.551069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inverting `wakeup` and `onset` events","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"event\"] = submission[\"event\"].replace({\"onset\":\"wakeup\",\"wakeup\":\"onset\"})\nsubmission[\"score\"] = 1\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:24.556911Z","iopub.execute_input":"2023-09-16T17:09:24.557220Z","iopub.status.idle":"2023-09-16T17:09:24.837945Z","shell.execute_reply.started":"2023-09-16T17:09:24.557193Z","shell.execute_reply":"2023-09-16T17:09:24.836672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Randomly shuffle the rows","metadata":{}},{"cell_type":"code","source":"submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\nsubmission[\"score\"] = 1\nsubmission = submission.sample(frac=1)\n\n# take a look\nsubmission.head(15)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:24.839389Z","iopub.execute_input":"2023-09-16T17:09:24.839822Z","iopub.status.idle":"2023-09-16T17:09:24.861589Z","shell.execute_reply.started":"2023-09-16T17:09:24.839784Z","shell.execute_reply":"2023-09-16T17:09:24.860375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"event_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:24.862897Z","iopub.execute_input":"2023-09-16T17:09:24.863235Z","iopub.status.idle":"2023-09-16T17:09:25.139550Z","shell.execute_reply.started":"2023-09-16T17:09:24.863199Z","shell.execute_reply":"2023-09-16T17:09:25.138445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using two `series_id`","metadata":{}},{"cell_type":"code","source":"\n# extract two series to be the \"ground truth\"\nsolution =  train_events.loc[(train_events[\"series_id\"] == \"d515236bdeec\") | (train_events[\"series_id\"] == \"d5e47b94477e\")].copy()\n\n# create a perfect submission\nperfect_submission = solution[[\"series_id\",\"step\",\"event\"]].copy()\n\n# create a perfect score column\nperfect_submission[\"score\"] = 1\n\nevent_detection_ap(solution, perfect_submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:25.141048Z","iopub.execute_input":"2023-09-16T17:09:25.141748Z","iopub.status.idle":"2023-09-16T17:09:25.802850Z","shell.execute_reply.started":"2023-09-16T17:09:25.141687Z","shell.execute_reply":"2023-09-16T17:09:25.801681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Randomly shuffle the two series ","metadata":{}},{"cell_type":"code","source":"# create a perfect submission\nsubmission = solution[[\"series_id\",\"step\",\"event\"]].copy()\n\n# create a perfect score column\nsubmission[\"score\"] = 1\n\nsubmission = submission.sample(frac=1)\n\nevent_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:25.804844Z","iopub.execute_input":"2023-09-16T17:09:25.805626Z","iopub.status.idle":"2023-09-16T17:09:26.460090Z","shell.execute_reply.started":"2023-09-16T17:09:25.805587Z","shell.execute_reply":"2023-09-16T17:09:26.458776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Two series, different `score` values","metadata":{}},{"cell_type":"code","source":"series_1 =  train_events.loc[(train_events[\"series_id\"] == \"d515236bdeec\")].copy()\nseries_1[\"score\"] = 1\n\nseries_2 =  train_events.loc[(train_events[\"series_id\"] == \"d5e47b94477e\")].copy()\nseries_2[\"score\"] = 0.1\n\nsubmission = pd.concat([series_1, series_2], axis=0)\n\nprint(event_detection_ap(solution, submission, tolerances))","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:26.461618Z","iopub.execute_input":"2023-09-16T17:09:26.462283Z","iopub.status.idle":"2023-09-16T17:09:27.148766Z","shell.execute_reply.started":"2023-09-16T17:09:26.462243Z","shell.execute_reply":"2023-09-16T17:09:27.147720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Two series, same `score` (`1`), one with shifted `step`","metadata":{}},{"cell_type":"code","source":"solution_1 =  train_events.loc[(train_events[\"series_id\"] == \"d515236bdeec\")].copy()\nseries_1_submission = solution_1[[\"series_id\",\"step\",\"event\"]].copy()\nseries_1_submission[\"score\"] = 1\n\n\nsolution_2 =  train_events.loc[(train_events[\"series_id\"] == \"d5e47b94477e\")].copy()\nseries_2_submission = solution_2[[\"series_id\",\"step\",\"event\"]].copy()\nseries_2_submission[\"score\"] = 1\nseries_2_submission[\"step\"] = series_2_submission[\"step\"]+350\n\n\nsubmission = pd.concat([series_1_submission, series_2_submission], axis=0)\n\nprint(event_detection_ap(solution, submission, tolerances))","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:27.150323Z","iopub.execute_input":"2023-09-16T17:09:27.150760Z","iopub.status.idle":"2023-09-16T17:09:27.833217Z","shell.execute_reply.started":"2023-09-16T17:09:27.150700Z","shell.execute_reply":"2023-09-16T17:09:27.831948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Two series, one with shifted `step` and lower `score` (`0.1`)","metadata":{}},{"cell_type":"code","source":"solution_1 =  train_events.loc[(train_events[\"series_id\"] == \"d515236bdeec\")].copy()\nseries_1_submission = solution_1[[\"series_id\",\"step\",\"event\"]].copy()\nseries_1_submission[\"score\"] = 1\n\n\nsolution_2 =  train_events.loc[(train_events[\"series_id\"] == \"d5e47b94477e\")].copy()\nseries_2_submission = solution_2[[\"series_id\",\"step\",\"event\"]].copy()\nseries_2_submission[\"score\"] = 0.1\nseries_2_submission[\"step\"] = series_2_submission[\"step\"]+350\n\n\nsubmission = pd.concat([series_1_submission, series_2_submission], axis=0)\n\nprint(event_detection_ap(solution, submission, tolerances))","metadata":{"execution":{"iopub.status.busy":"2023-09-16T17:09:27.834983Z","iopub.execute_input":"2023-09-16T17:09:27.835389Z","iopub.status.idle":"2023-09-16T17:09:28.510769Z","shell.execute_reply.started":"2023-09-16T17:09:27.835349Z","shell.execute_reply":"2023-09-16T17:09:28.509656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Related notebooks\n* [\"Competition Metric - DFL Event Detection AP](https://www.kaggle.com/code/ryanholbrook/competition-metric-dfl-event-detection-ap)","metadata":{}}]}