{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 🧐 Supporter of Understanding Submission Metric\n---\n[Objective]\n- In order to understanding submission metric, we can try submission metric on Jupyter-notebook.\n\n[References]\n- https://www.kaggle.com/competitions/child-mind-institute-detect-sleep-states\n- https://www.kaggle.com/code/metric/event-detection-ap?scriptVersionId=149565357\n","metadata":{}},{"cell_type":"markdown","source":"# 0. Import libraries","metadata":{}},{"cell_type":"code","source":"# =================================================\n# Import libraries\n# =================================================\nimport numpy as np\nimport pandas as pd\nimport pandas.api.types\nfrom typing import Dict, List, Tuple","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:02.801552Z","iopub.execute_input":"2023-11-19T06:37:02.801848Z","iopub.status.idle":"2023-11-19T06:37:03.124964Z","shell.execute_reply.started":"2023-11-19T06:37:02.801821Z","shell.execute_reply":"2023-11-19T06:37:03.123651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_df(df:pd.DataFrame, num:int=10, tail:bool=False)-> None:\n    \"\"\"show shape and head of df \n\n    Args:\n        df (pd.DataFrame): data\n        num (int, optional): length of raws. Defaults to 3:int.\n        tail (bool, optional): if True show tail. Defaults to False:bool.\n    \"\"\"\n    print(df.shape)\n    display(df.head(num))\n    if tail: \n        display(df.tail(num))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.127573Z","iopub.execute_input":"2023-11-19T06:37:03.128096Z","iopub.status.idle":"2023-11-19T06:37:03.134455Z","shell.execute_reply.started":"2023-11-19T06:37:03.128061Z","shell.execute_reply":"2023-11-19T06:37:03.133053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Ground truth of train event","metadata":{}},{"cell_type":"code","source":"df_event = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nshow_df(df_event)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.135719Z","iopub.execute_input":"2023-11-19T06:37:03.136158Z","iopub.status.idle":"2023-11-19T06:37:03.198696Z","shell.execute_reply.started":"2023-11-19T06:37:03.136128Z","shell.execute_reply":"2023-11-19T06:37:03.197956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Load Submission Metric\n---\ncf.) Event Detection AP(ver.8)  \nhttps://www.kaggle.com/code/metric/event-detection-ap?scriptVersionId=149565357","metadata":{}},{"cell_type":"code","source":"class ParticipantVisibleError(Exception):\n    # If you want an error message to be shown to participants, you must raise the error as a ParticipantVisibleError\n    # All other errors will only be shown to the competition host. This helps prevent unintentional leakage of solution data.\n    pass","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-19T06:37:03.200636Z","iopub.execute_input":"2023-11-19T06:37:03.201584Z","iopub.status.idle":"2023-11-19T06:37:03.206283Z","shell.execute_reply.started":"2023-11-19T06:37:03.201529Z","shell.execute_reply":"2023-11-19T06:37:03.205385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n) -> float:\n    \"\"\"Event Detection Average Precision, an AUCPR metric for event detection in time series and video.\n\n    This metric is similar to IOU-threshold average precision metrics commonly used in object detection. \n    For events occuring in time series, we replace the IOU threshold with a time tolerance.\n\n    Submissions are evaluated on the average precision of detected events,\n    averaged over timestamp error tolerance thresholds, averaged over event classes.\n\n    Detections are matched to ground-truth events within error tolerances, with ambiguities resolved in order of decreasing confidence.\n\n    Detailed Description\n    --------------------\n    Evaluation proceeds in four steps:\n\n    1. Selection - (optional) Predictions not within a series' scoring intervals are dropped.\n    2. Assignment - Predicted events are matched with ground-truth events.\n    3. Scoring - Each group of predictions is scored against its corresponding group of ground-truth events via Average Precision.\n    4. Reduction - The multiple AP scores are averaged to produce a single overall score.\n\n    <Selection>\n\n    With each series there may be a defined set of scoring intervals giving the intervals of time over \n    which zero or more ground-truth events might be annotated in that series. \n    A prediction will be evaluated only if it falls within a scoring interval. \n    These scoring intervals can be chosen to improve the fairness of evaluation by, for instance, ignoring edge-cases or ambiguous events.\n\n    It is recommended that, if used, scoring intervals be provided for training data but not test data.\n\n    <Assignment>\n\n    For each set of predictions and ground-truths within the same `event x tolerance x series_id` group, \n    we match each ground-truth to the highest-confidence unmatched prediction occurring within the allowed tolerance.\n\n    Some ground-truths may not be matched to a prediction and some predictions may not be matched to a ground-truth. \n    They will still be accounted for in the scoring, however.\n\n    <Scoring>\n\n    Collecting the events within each `series_id`, we compute an Average Precision score for each `event x tolerance` group. \n    The average precision score is the area under the (step-wise) precision-recall curve generated by decreasing confidence score thresholds over the predictions. \n    In this calculation, matched predictions over the threshold are scored as TP and unmatched predictions as FP. \n    Unmatched ground-truths are scored as FN.\n\n    <Reduction>\n\n    The final score is the average of the above AP scores, first averaged over tolerance, then over event.\n\n    <Parameters>\n    ----------\n    solution : pd.DataFrame, with columns:\n        `series_id_column_name` identifier for each time series\n        `time_column_name` the time of occurence for each event as a numeric type\n        `event_column_name` class label for each event\n\n        The solution contains the time of occurence of one or more types of event within one or more time series. \n        The metric expects the solution to contain the same event types as those given in `tolerances`.\n\n        When `use_scoring_intervals == True`, you may include `start` and `end` events to delimit intervals within which detections will be scored.\n        Detected events (from the user submission) outside of these events will be ignored.\n\n    submission : pd.DataFrame, with columns as above and in addition:\n        `score_column_name` the predicted confidence score for the detected event\n\n    tolerances : Dict[str, List[float]]\n        Maps each event class to a list of timestamp tolerances used for matching detections to ground-truth events.\n\n    use_scoring_intervals: bool, default False\n        Whether to ignore predicted events outside intervals delimited by `'start'` and `'end'` events in the solution. \n        When `False`, the solution should not include `'start'` and `'end'` events.\n        See the examples for illustration.\n\n    Returns\n    -------\n    event_detection_ap : float\n        The mean average precision of the detected events.\n\n    Examples\n    --------\n    Detecting `'pass'` events in football:\n    >>> column_names = {\n    ...     'series_id_column_name': 'video_id',\n    ...     'time_column_name': 'time',\n    ...     'event_column_name': 'event',\n    ...     'score_column_name': 'score',\n    ... }\n    >>> tolerances = {'pass': [1.0]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a'],\n    ...     'event': ['pass', 'pass'],\n    ...     'time': [0, 15],\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a'],\n    ...     'event': ['pass', 'pass', 'pass'],\n    ...     'score': [1.0, 0.5, 1.0],\n    ...     'time': [0, 10, 14.5],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names)\n    1.0\n\n    Increasing the confidence score of the false detection above the true detections decreases the AP.\n    >>> submission.loc[1, 'score'] = 1.5\n    >>> score(solution, submission, tolerances, **column_names)\n    0.6666666666666666...\n\n    Likewise, decreasing the confidence score of a true detection below the false detection also decreases the AP.\n    >>> submission.loc[1, 'score'] = 0.5  # reset\n    >>> submission.loc[0, 'score'] = 0.0\n    >>> score(solution, submission, tolerances, **column_names)\n    0.8333333333333333...\n\n    We average AP scores over tolerances. Previously, the detection at 14.5 would match, but adding smaller tolerances gives AP scores \n    where it does not match. \n    This results in both a FN, since the ground-truth wasn't detected, and a FP, since the detected event matches no ground-truth.\n    >>> tolerances = {'pass': [0.1, 0.2, 1.0]}\n    >>> score(solution, submission, tolerances, **column_names)\n    0.3888888888888888...\n\n    We also average over time series and over event classes.\n    >>> tolerances = {'pass': [0.5, 1.0], 'challenge': [0.25, 0.50]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'b'],\n    ...     'event': ['pass', 'challenge', 'pass'],\n    ...     'time': [0, 15, 0],  # restart time for new time series b\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'b'],\n    ...     'event': ['pass', 'challenge', 'pass'],\n    ...     'score': [1.0, 0.5, 1.0],\n    ...     'time': [0, 15, 0],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names)\n    1.0\n\n    By adding scoring intervals to the solution, we may choose to ignore detections outside of those intervals.\n    >>> tolerances = {'pass': [1.0]}\n    >>> solution = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a', 'a'],\n    ...     'event': ['start', 'pass', 'pass', 'end'],\n    ...     'time': [0, 10, 20, 30],\n    ... })\n    >>> submission = pd.DataFrame({\n    ...     'video_id': ['a', 'a', 'a'],\n    ...     'event': ['pass', 'pass', 'pass'],\n    ...     'score': [1.0, 1.0, 1.0],\n    ...     'time': [10, 20, 40],\n    ... })\n    >>> score(solution, submission, tolerances, **column_names, use_scoring_intervals=True)\n    1.0\n\n    \"\"\"\n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{column_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.207591Z","iopub.execute_input":"2023-11-19T06:37:03.208346Z","iopub.status.idle":"2023-11-19T06:37:03.222211Z","shell.execute_reply.started":"2023-11-19T06:37:03.208314Z","shell.execute_reply":"2023-11-19T06:37:03.220969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_detections(\n        detections: pd.DataFrame, intervals: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Drop detections not inside a scoring interval.\"\"\"\n    detection_time = detections.loc[:, time_column_name].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.\n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.225535Z","iopub.execute_input":"2023-11-19T06:37:03.226245Z","iopub.status.idle":"2023-11-19T06:37:03.236034Z","shell.execute_reply.started":"2023-11-19T06:37:03.226217Z","shell.execute_reply":"2023-11-19T06:37:03.235074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.237298Z","iopub.execute_input":"2023-11-19T06:37:03.237558Z","iopub.status.idle":"2023-11-19T06:37:03.249693Z","shell.execute_reply.started":"2023-11-19T06:37:03.237534Z","shell.execute_reply":"2023-11-19T06:37:03.248627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\n\n","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.250832Z","iopub.execute_input":"2023-11-19T06:37:03.251122Z","iopub.status.idle":"2023-11-19T06:37:03.261934Z","shell.execute_reply.started":"2023-11-19T06:37:03.251095Z","shell.execute_reply":"2023-11-19T06:37:03.261044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.265113Z","iopub.execute_input":"2023-11-19T06:37:03.265430Z","iopub.status.idle":"2023-11-19T06:37:03.276322Z","shell.execute_reply.started":"2023-11-19T06:37:03.265406Z","shell.execute_reply":"2023-11-19T06:37:03.275300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    series_ids = set(ground_truths[series_id_column_name]).union(submission[series_id_column_name])\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, sid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for sid in series_ids],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.280300Z","iopub.execute_input":"2023-11-19T06:37:03.281087Z","iopub.status.idle":"2023-11-19T06:37:03.294799Z","shell.execute_reply.started":"2023-11-19T06:37:03.281055Z","shell.execute_reply":"2023-11-19T06:37:03.293711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Try Example\n---\n- If we prepare solution & submission example(pd.DataFrame), we can try to use metric to understand it.\n- Change number of each columns -> try scoring may be support our understanding.\n\nthe example is footboll event (pass, charange, throw-In) detection.  \nhttps://www.kaggle.com/competitions/dfl-bundesliga-data-shootout","metadata":{}},{"cell_type":"code","source":"# =================================================\n# Set some placeholders for global parameters\n# =================================================\nseries_id_column_name = \"series_id\" # None\ntime_column_name      = \"step\" # None\nevent_column_name     = \"event\" # None\nscore_column_name     = \"score\" # None\nuse_scoring_intervals = None","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.296185Z","iopub.execute_input":"2023-11-19T06:37:03.296497Z","iopub.status.idle":"2023-11-19T06:37:03.309962Z","shell.execute_reply.started":"2023-11-19T06:37:03.296467Z","shell.execute_reply":"2023-11-19T06:37:03.309139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =================================================\n# Set example\n# =================================================\ncolumn_names = {\n    'series_id_column_name': 'video_id',\n    'time_column_name': 'time',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n    }\ntolerances = {'pass': [1.0]}\nsolution = pd.DataFrame({\n    'video_id': ['a', 'a'],\n    'event': ['pass', 'pass'],\n    'time': [0, 15],\n    })\nsubmission = pd.DataFrame({\n    'video_id': ['a', 'a', 'a'],\n    'event': ['pass', 'pass', 'pass'],\n    'score': [1.0, 0.5, 1.0],\n    'time': [0, 10, 14.5],\n    })\n\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.312843Z","iopub.execute_input":"2023-11-19T06:37:03.313447Z","iopub.status.idle":"2023-11-19T06:37:03.394685Z","shell.execute_reply.started":"2023-11-19T06:37:03.313404Z","shell.execute_reply":"2023-11-19T06:37:03.394040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nIncreasing the confidence score of the false detection above the true detections decreases the AP.\n>>> submission.loc[1, 'score'] = 1.5\n>>> score(solution, submission, tolerances, **column_names)\n0.6666666666666666...\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.395758Z","iopub.execute_input":"2023-11-19T06:37:03.396695Z","iopub.status.idle":"2023-11-19T06:37:03.401539Z","shell.execute_reply.started":"2023-11-19T06:37:03.396670Z","shell.execute_reply":"2023-11-19T06:37:03.400937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.loc[1, 'score'] = 1.5\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.402471Z","iopub.execute_input":"2023-11-19T06:37:03.402998Z","iopub.status.idle":"2023-11-19T06:37:03.447988Z","shell.execute_reply.started":"2023-11-19T06:37:03.402974Z","shell.execute_reply":"2023-11-19T06:37:03.447081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"    \nLikewise, decreasing the confidence score of a true detection below the false detection also decreases the AP.\n>>> submission.loc[1, 'score'] = 0.5  # reset\n>>> submission.loc[0, 'score'] = 0.0\n>>> score(solution, submission, tolerances, **column_names)\n0.8333333333333333...\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.451600Z","iopub.execute_input":"2023-11-19T06:37:03.452384Z","iopub.status.idle":"2023-11-19T06:37:03.461135Z","shell.execute_reply.started":"2023-11-19T06:37:03.452358Z","shell.execute_reply":"2023-11-19T06:37:03.460052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.loc[1, 'score'] = 0.5  # reset\nsubmission.loc[0, 'score'] = 0.0\n\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.465275Z","iopub.execute_input":"2023-11-19T06:37:03.465574Z","iopub.status.idle":"2023-11-19T06:37:03.509048Z","shell.execute_reply.started":"2023-11-19T06:37:03.465549Z","shell.execute_reply":"2023-11-19T06:37:03.508197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nWe average AP scores over tolerances. Previously, the detection at 14.5 would match, but adding smaller tolerances gives AP scores \nwhere it does not match. \nThis results in both a FN, since the ground-truth wasn't detected, and a FP, since the detected event matches no ground-truth.\n>>> tolerances = {'pass': [0.1, 0.2, 1.0]}\n>>> score(solution, submission, tolerances, **column_names)\n0.3888888888888888...\n\nWe also average over time series and over event classes.\n>>> tolerances = {'pass': [0.5, 1.0], 'challenge': [0.25, 0.50]}\n>>> solution = pd.DataFrame({\n...     'video_id': ['a', 'a', 'b'],\n...     'event': ['pass', 'challenge', 'pass'],\n...     'time': [0, 15, 0],  # restart time for new time series b\n... })\n>>> submission = pd.DataFrame({\n...     'video_id': ['a', 'a', 'b'],\n...     'event': ['pass', 'challenge', 'pass'],\n...     'score': [1.0, 0.5, 1.0],\n...     'time': [0, 15, 0],\n... })\n>>> score(solution, submission, tolerances, **column_names)\n1.0\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.510171Z","iopub.execute_input":"2023-11-19T06:37:03.511422Z","iopub.status.idle":"2023-11-19T06:37:03.518727Z","shell.execute_reply.started":"2023-11-19T06:37:03.511369Z","shell.execute_reply":"2023-11-19T06:37:03.517721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tolerances = {'pass': [0.5, 1.0], 'challenge': [0.25, 0.50]}\nsolution = pd.DataFrame({\n     'video_id': ['a', 'a', 'b'],\n     'event': ['pass', 'challenge', 'pass'],\n     'time': [0, 15, 0],  # restart time for new time series b\n})\nsubmission = pd.DataFrame({\n     'video_id': ['a', 'a', 'b'],\n     'event': ['pass', 'challenge', 'pass'],\n     'score': [1.0, 0.5, 1.0],\n     'time': [0, 15, 0],\n})\n\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.520149Z","iopub.execute_input":"2023-11-19T06:37:03.521456Z","iopub.status.idle":"2023-11-19T06:37:03.586758Z","shell.execute_reply.started":"2023-11-19T06:37:03.521412Z","shell.execute_reply":"2023-11-19T06:37:03.585749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nBy adding scoring intervals to the solution, we may choose to ignore detections outside of those intervals.\n>>> tolerances = {'pass': [1.0]}\n>>> solution = pd.DataFrame({\n...     'video_id': ['a', 'a', 'a', 'a'],\n...     'event': ['start', 'pass', 'pass', 'end'],\n...     'time': [0, 10, 20, 30],\n... })\n>>> submission = pd.DataFrame({\n...     'video_id': ['a', 'a', 'a'],\n...     'event': ['pass', 'pass', 'pass'],\n...     'score': [1.0, 1.0, 1.0],\n...     'time': [10, 20, 40],\n... })\n>>> score(solution, submission, tolerances, **column_names, use_scoring_intervals=True)\n1.0\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.587802Z","iopub.execute_input":"2023-11-19T06:37:03.588777Z","iopub.status.idle":"2023-11-19T06:37:03.594824Z","shell.execute_reply.started":"2023-11-19T06:37:03.588749Z","shell.execute_reply":"2023-11-19T06:37:03.593974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tolerances = {'pass': [1.0]}\nsolution = pd.DataFrame({\n     'video_id': ['a', 'a', 'a', 'a'],\n     'event': ['start', 'pass', 'pass', 'end'],\n     'time': [0, 10, 20, 30],\n})\nsubmission = pd.DataFrame({\n     'video_id': ['a', 'a', 'a'],\n     'event': ['pass', 'pass', 'pass'],\n     'score': [1.0, 1.0, 1.0],\n     'time': [10, 20, 40],\n})\n\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.596328Z","iopub.execute_input":"2023-11-19T06:37:03.596868Z","iopub.status.idle":"2023-11-19T06:37:03.644585Z","shell.execute_reply.started":"2023-11-19T06:37:03.596837Z","shell.execute_reply":"2023-11-19T06:37:03.643683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. In case of this competition","metadata":{}},{"cell_type":"code","source":"solution = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nshow_df(solution)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.645763Z","iopub.execute_input":"2023-11-19T06:37:03.646133Z","iopub.status.idle":"2023-11-19T06:37:03.670168Z","shell.execute_reply.started":"2023-11-19T06:37:03.646103Z","shell.execute_reply":"2023-11-19T06:37:03.669330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution[[\"series_id\", \"event\", \"step\"]].copy()\nsubmission[\"score\"] = 1.0\nshow_df(submission)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.671282Z","iopub.execute_input":"2023-11-19T06:37:03.672220Z","iopub.status.idle":"2023-11-19T06:37:03.688795Z","shell.execute_reply.started":"2023-11-19T06:37:03.672188Z","shell.execute_reply":"2023-11-19T06:37:03.687712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# =================================================\n# Set some placeholders for global parameters\n# =================================================\nseries_id_column_name = \"series_id\" # None\ntime_column_name      = \"step\" # None\nevent_column_name     = \"event\" # None\nscore_column_name     = \"score\" # None\nuse_scoring_intervals = None","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.689981Z","iopub.execute_input":"2023-11-19T06:37:03.690416Z","iopub.status.idle":"2023-11-19T06:37:03.695527Z","shell.execute_reply.started":"2023-11-19T06:37:03.690307Z","shell.execute_reply":"2023-11-19T06:37:03.694468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score',\n}\ntolerances = {\n    'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]\n }\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:37:03.696753Z","iopub.execute_input":"2023-11-19T06:37:03.696989Z","iopub.status.idle":"2023-11-19T06:38:05.759846Z","shell.execute_reply.started":"2023-11-19T06:37:03.696967Z","shell.execute_reply":"2023-11-19T06:38:05.758343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"solution.dropna(how=\"any\", axis=0, inplace=True)\nsubmission.dropna(how=\"any\", axis=0, inplace=True)\n\nsubmission[\"score\"] = 1.0\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:38:05.761221Z","iopub.execute_input":"2023-11-19T06:38:05.761789Z","iopub.status.idle":"2023-11-19T06:39:05.492231Z","shell.execute_reply.started":"2023-11-19T06:38:05.761753Z","shell.execute_reply":"2023-11-19T06:39:05.491589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution.copy()\nsubmission[\"score\"] = 1.0\nsubmission[\"step\"] -= 15\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution.copy()\nsubmission[\"score\"] = 1.0\nsubmission[\"step\"] -= 40\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:39:05.493436Z","iopub.execute_input":"2023-11-19T06:39:05.494186Z","iopub.status.idle":"2023-11-19T06:40:08.537720Z","shell.execute_reply.started":"2023-11-19T06:39:05.494161Z","shell.execute_reply":"2023-11-19T06:40:08.536391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = solution.copy()\nsubmission[\"score\"] = 0.5\nprint(\"=\"*80); print(\"solution\"); print(\"=\"*80); show_df(solution)\nprint(\"=\"*80); print(\"submission\"); print(\"=\"*80); show_df(submission)\n\nprint(\">> Score\")\nscore(solution, submission, tolerances, **column_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-19T06:40:08.539177Z","iopub.execute_input":"2023-11-19T06:40:08.539474Z","iopub.status.idle":"2023-11-19T06:41:11.663131Z","shell.execute_reply.started":"2023-11-19T06:40:08.539448Z","shell.execute_reply":"2023-11-19T06:41:11.662039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}