{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## CMI 2023 - AP metric and score","metadata":{}},{"cell_type":"markdown","source":"Use the competition AP metric to evalute matched onsets and wakeups from one series' data and compare it with a simple way to evaluate the metric that works when all score values are 1.0 (or another constant.)\n\nThe Precision-Recall curve used to calculate the AP uses thresholds of the `score` value. The AP is calculated for matches in each event X tolerance combination; so 20 APs are calculated from 20 P-R curves. These 20 values are averaged to give the final Average Precision metric value.\n\nIf the `score` is a constant then the P-R \"curve\" is a single point (plus the starting point of P=1, R=0) and the Average Precision is easy to calculate as:\n```\n   AP for this event X tolerance = Precision * Recall =\n        (TPs / Number of actual events) * (TPs / Number predicted events)\n```\nwhere TPs is the number of correct matches in the event X tolerance selection. THis calculation is compared the the AP metric result below.\n","metadata":{}},{"cell_type":"markdown","source":"### Things to use","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-22T19:32:39.710235Z","iopub.execute_input":"2023-09-22T19:32:39.710632Z","iopub.status.idle":"2023-09-22T19:32:40.115839Z","shell.execute_reply.started":"2023-09-22T19:32:39.710598Z","shell.execute_reply":"2023-09-22T19:32:40.114938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### The Event Detection AP metric code","metadata":{}},{"cell_type":"code","source":"# The Event Detection Average Precision metric\n# from: https://www.kaggle.com/code/metric/event-detection-ap/notebook\n# Removed some comments etc.\n# and added a few print statements to show intermediate values.\n\nimport pandas.api.types\nfrom typing import Dict, List, Tuple\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n# Set some placeholders for global parameters\nseries_id_column_name = None\ntime_column_name = None\nevent_column_name = None\nscore_column_name = None\nuse_scoring_intervals = None\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.117560Z","iopub.execute_input":"2023-09-22T19:32:40.118346Z","iopub.status.idle":"2023-09-22T19:32:40.123962Z","shell.execute_reply.started":"2023-09-22T19:32:40.118313Z","shell.execute_reply":"2023-09-22T19:32:40.123146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The top-level routine\n# Does various checks and then calls event_detection_ap() for the real work.\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n                                    ) -> float:\n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == \\\n            set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{target_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.125211Z","iopub.execute_input":"2023-09-22T19:32:40.125526Z","iopub.status.idle":"2023-09-22T19:32:40.139432Z","shell.execute_reply.started":"2023-09-22T19:32:40.125498Z","shell.execute_reply":"2023-09-22T19:32:40.138372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_detections(\n        detections: pd.DataFrame, intervals: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Drop detections not inside a scoring interval.\"\"\"\n    detection_time = detections.loc[:, time_column_name].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.\n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.141902Z","iopub.execute_input":"2023-09-22T19:32:40.142336Z","iopub.status.idle":"2023-09-22T19:32:40.155649Z","shell.execute_reply.started":"2023-09-22T19:32:40.142299Z","shell.execute_reply":"2023-09-22T19:32:40.154658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.156872Z","iopub.execute_input":"2023-09-22T19:32:40.157186Z","iopub.status.idle":"2023-09-22T19:32:40.173795Z","shell.execute_reply.started":"2023-09-22T19:32:40.157160Z","shell.execute_reply":"2023-09-22T19:32:40.172594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.175369Z","iopub.execute_input":"2023-09-22T19:32:40.175995Z","iopub.status.idle":"2023-09-22T19:32:40.186702Z","shell.execute_reply.started":"2023-09-22T19:32:40.175940Z","shell.execute_reply":"2023-09-22T19:32:40.185496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int\n                           ) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    \n    # dd added:  **************************************************\n    ##print(\"average_precision_score():\")\n    ##print(\"   Recall:\", recall, \" Precision:\", precision)\n    #  ************************************************************\n    \n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.188733Z","iopub.execute_input":"2023-09-22T19:32:40.189228Z","iopub.status.idle":"2023-09-22T19:32:40.202007Z","shell.execute_reply.started":"2023-09-22T19:32:40.189184Z","shell.execute_reply":"2023-09-22T19:32:40.201277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    \n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    # dd added:  **************************************************\n    print(\"\\n\\nevent_detection_ap():\")\n    print(\"\\n   The onset APs by tolerance:\")\n    print(ap_table['onset'])\n    print(\"\\n   The wakeup APs by tolerance:\")\n    print(ap_table['wakeup'])\n    print(\"\\n   The AP for each class:\")\n    print(ap_table.groupby(event_column_name).mean())\n    #  ************************************************************    \n    \n    return mean_ap\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.203323Z","iopub.execute_input":"2023-09-22T19:32:40.203998Z","iopub.status.idle":"2023-09-22T19:32:40.226574Z","shell.execute_reply.started":"2023-09-22T19:32:40.203967Z","shell.execute_reply":"2023-09-22T19:32:40.224640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using the Event Detection AP metric","metadata":{}},{"cell_type":"code","source":"# Some of the comments:\ndummy = \"\"\"\n    Submissions are evaluated on the average precision of detected events,\n    averaged over timestamp error tolerance thresholds, averaged over event\n    classes.\n\n    Detections are matched to ground-truth events within error tolerances, with\n    ambiguities resolved in order of decreasing confidence [aka \"score\"].\n    \n    Scoring\n\n    Collecting the events within each `series_id`, we compute an Average\n    Precision score for each `event x tolerance` group. The average precision\n    score is the area under the (step-wise) precision-recall curve generated by\n    decreasing confidence _score_ thresholds over the predictions. In this\n    calculation, matched predictions over the threshold are scored as TP and\n    unmatched predictions as FP. Unmatched ground-truths are scored as FN.\n    \n    [dd: For the case where score=1.0 for all events,\n    then the AP for an `event x tolerance` group is just:\n        AP for event x tolerance = Precision * Recall =\n            (TPs / Number of actual events) * (TPs / Number predicted events)\n    These 20 APs are then averaged to get the final AP metric value.]\n\n    Reduction\n\n    The final score is the average of the above AP scores, first averaged over\n    tolerance, then over event.\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.227858Z","iopub.execute_input":"2023-09-22T19:32:40.228625Z","iopub.status.idle":"2023-09-22T19:32:40.241838Z","shell.execute_reply.started":"2023-09-22T19:32:40.228592Z","shell.execute_reply":"2023-09-22T19:32:40.240810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Example using data from one series, '0a96f4993bd7', in the CMI train data\n\ncolumn_names = {\n     'series_id_column_name': 'series_id',\n     'time_column_name': 'step',\n     'event_column_name': 'event',\n     'score_column_name': 'score',\n        }\n\ntolerances = {'onset': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n                'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360], }\n\n# The Ground Truth\n# Values from analysis of series id 0a96f4993bd7\nactual_onsets  = [  8256,  26460,  45156,  60816,  77100,\n                      94764, 112044, 129588, 148092, 167196,\n                          182556, 198804, 215676, 232464]\n                        ##250000, 270000, 290000, 31000]  # 4 extra = FNs\nactual_wakeups = [ 15024,  33972,  51336,  67176,  83964,\n                    100980, 118752, 135972, 155412, 172392,\n                        188268, 205596, 222660, 239712]\n                     ##260000, 280000, 300000, 320000]  # 4 extra = FNs\nsolution = pd.DataFrame({\n            'series_id': 28*['0a96f4993bd7'],\n            'event': 14*['onset'] + 14*['wakeup'],  # concat lists\n            'step': actual_onsets + actual_wakeups,\n            })\n\n# The Model Predictions\npredicted_onsets = [  8179,  26026,  45015,  60924,  77071,\n                        94680, 112029, 129374, 147544, 164156,\n                         181713, 198523, 215133, 232417]\n\npredicted_wakeups = [ 14992,  34007,  51318,  67158,  83993,\n                   101044, 118790, 136057, 155423, 172011,\n                    187978, 205543, 222669, 239721]\n\n\nsubmission = pd.DataFrame({\n            'series_id': 28*['0a96f4993bd7'],\n            'event': 14*['onset'] + 14*['wakeup'],\n            'score': 28*[1.0],\n            'step': predicted_onsets + predicted_wakeups,\n            })","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.247084Z","iopub.execute_input":"2023-09-22T19:32:40.247901Z","iopub.status.idle":"2023-09-22T19:32:40.262120Z","shell.execute_reply.started":"2023-09-22T19:32:40.247856Z","shell.execute_reply":"2023-09-22T19:32:40.260919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Score it\nfinal_ap = score(solution, submission, tolerances, **column_names)\nprint(\"\\nThe overall AP = {:.3f}\".format(final_ap),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.263983Z","iopub.execute_input":"2023-09-22T19:32:40.264745Z","iopub.status.idle":"2023-09-22T19:32:40.497719Z","shell.execute_reply.started":"2023-09-22T19:32:40.264704Z","shell.execute_reply":"2023-09-22T19:32:40.496605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## A simpler metric evaluation routine","metadata":{}},{"cell_type":"markdown","source":"This gives the AP assuming all the score values are constant, e.g., = 1.0\n\nThe outputs below are the same as the competition metric values for score=1.0  :)","metadata":{}},{"cell_type":"code","source":"# Calculate the AP for matched values from one series id\n# for the case when score is a constant.\ndef calc_AP_values(vals_pred, vals_true):\n    # These countings will handle NaN values OK\n    num_pred_pos = max([sum(abs(vals_pred) > -1), 1])\n    num_actual_pos = max([sum(abs(vals_true) > -1), 1])\n    tol_levels = [12,36,60,90,120,150,180,240,300,360]\n    AP_values = np.zeros(len(tol_levels))\n    for itol, tol_level in enumerate(tol_levels):\n        num_TP = sum(abs(vals_pred-vals_true) < tol_level)\n        prec = num_TP/num_pred_pos\n        recall = num_TP/num_actual_pos\n        # Average-precision assuming all scores are the same, e.g., 1.0,\n        # so there is only one point in the P-R curve (plus the recall=0, prec=1 point.)\n        AP_values[itol] = prec * recall\n    # Returns values for each tolerance level\n    return AP_values","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:34:34.680356Z","iopub.execute_input":"2023-09-22T19:34:34.680779Z","iopub.status.idle":"2023-09-22T19:34:34.689038Z","shell.execute_reply.started":"2023-09-22T19:34:34.680745Z","shell.execute_reply":"2023-09-22T19:34:34.687925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onset_APs = calc_AP_values(np.array(predicted_onsets), \n               np.array(actual_onsets))\nonset_APs","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:34:35.231014Z","iopub.execute_input":"2023-09-22T19:34:35.231762Z","iopub.status.idle":"2023-09-22T19:34:35.237988Z","shell.execute_reply.started":"2023-09-22T19:34:35.231729Z","shell.execute_reply":"2023-09-22T19:34:35.237282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wakeup_APs = calc_AP_values(np.array(predicted_wakeups), \n                               np.array(actual_wakeups))\nwakeup_APs","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:34:36.761474Z","iopub.execute_input":"2023-09-22T19:34:36.761835Z","iopub.status.idle":"2023-09-22T19:34:36.769429Z","shell.execute_reply.started":"2023-09-22T19:34:36.761804Z","shell.execute_reply":"2023-09-22T19:34:36.768529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"simple_ap = sum(onset_APs + wakeup_APs) / (len(onset_APs)+len(wakeup_APs))\nprint(\"\\nThe overall AP = {:.3f}\".format(final_ap),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:34:37.138191Z","iopub.execute_input":"2023-09-22T19:34:37.140855Z","iopub.status.idle":"2023-09-22T19:34:37.146510Z","shell.execute_reply.started":"2023-09-22T19:34:37.140816Z","shell.execute_reply":"2023-09-22T19:34:37.145470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The effect of score on the AP metric","metadata":{}},{"cell_type":"code","source":"# Modify some of the onset, wakeup scores.\n# Use 0.7 for cases where the region after onset\n# or the region before wakeup has more activity than some threshold\n# and so the determined onset or wakeup are likely to be unreliabe.\n# 3 nights have less reliable onsets: 9, 10, 11 [and are in error]\nsubmission.iloc[9-1,2] = 0.7\nsubmission.iloc[10-1,2] = 0.7\nsubmission.iloc[11-1,2] = 0.7\n# 1 night has unreliable wakeup: 9  [but it's actually OK]\nsubmission.iloc[9+14-1,2] = 0.7\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:36:22.149548Z","iopub.execute_input":"2023-09-22T19:36:22.150344Z","iopub.status.idle":"2023-09-22T19:36:22.171279Z","shell.execute_reply.started":"2023-09-22T19:36:22.150305Z","shell.execute_reply":"2023-09-22T19:36:22.169897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Score it\nfinal_ap = score(solution, submission, tolerances, **column_names)\nprint(\"\\nThe overall AP = {:.3f}\".format(final_ap),\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:36:31.238613Z","iopub.execute_input":"2023-09-22T19:36:31.239725Z","iopub.status.idle":"2023-09-22T19:36:31.470170Z","shell.execute_reply.started":"2023-09-22T19:36:31.239672Z","shell.execute_reply":"2023-09-22T19:36:31.468978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The three (properly) demoted onset scores increased the onset AP.   :)\n# The one (incorrectly) demoted wakeup score decreased the wakeup AP.     :(\n# So, there are risks and rewards in setting the score value.\n# However in this case the overall AP increased from 0.416 to 0.439. ","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.805757Z","iopub.execute_input":"2023-09-22T19:32:40.806167Z","iopub.status.idle":"2023-09-22T19:32:40.813004Z","shell.execute_reply.started":"2023-09-22T19:32:40.806138Z","shell.execute_reply":"2023-09-22T19:32:40.811937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<HR>","metadata":{}},{"cell_type":"markdown","source":"## Bonus: Example of Precision vs Recall and AP for a Classifier","metadata":{}},{"cell_type":"code","source":"# Confusion dots plot\ndef confusion_dots(y_true, y_pred, y_thresh = 0.5,\n                   title=\"Confusion Dots Plot\"):\n    plt.figure(figsize=(7,2.5))\n    plt.scatter(y_pred, -1*y_true+0.1*np.random.randn(len(y_true)), c='b', s=2)\n    plt.xlim(0.0,1.0); plt.ylim(-1.5,0.5)\n    plt.xlabel(\"Predicted Probabilty\")\n    plt.ylabel(\"1   <--  Class  -->   0\")\n    # Annotation of the regions and threshold\n    ythr2 = y_thresh/2.0\n    plt.plot([y_thresh, y_thresh],[-1.4,-0.6],color='gray')\n    plt.plot([y_thresh, y_thresh],[-0.4,0.4],color='gray')\n    plt.plot([0.02, 0.98],[-0.5,-0.5],color='gray')\n    plt.text(ythr2 - 0.03,       -1.40, \"FN\", fontsize=12, color='red')\n    plt.text(ythr2 + 0.5 - 0.03, -1.40, \"TP\", fontsize=12, color='green')\n    plt.text(ythr2 - 0.03,        0.30, \"TN\", fontsize=12, color='green')\n    plt.text(ythr2 + 0.5 - 0.03,  0.30, \"FP\", fontsize=12, color='red')\n    plt.title(title + \"    (threshold = \" + str(y_thresh) + \")\")\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.814191Z","iopub.execute_input":"2023-09-22T19:32:40.814488Z","iopub.status.idle":"2023-09-22T19:32:40.827256Z","shell.execute_reply.started":"2023-09-22T19:32:40.814462Z","shell.execute_reply":"2023-09-22T19:32:40.826368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate simulate y_true (0s and 1s) and y_pred (continuous 0 to 1)\nn_samples = 500\n\n# Generate random y_true values, 0 and 1 s\n# 0.25 will be 1\ny_true = 1*(np.random.random_sample(size=n_samples) > 0.75)\n\n# Generate y_pred values\ny_pred = ( np.clip(    # clip values to stay in 0 to 1 range\n            0.5 + # center at 0.5\n            y_true*0.40 - 0.2 + # assymetric offset from 0.5\n            # normal spread with different stds\n            (0.20+0.15*y_true)*np.random.randn(n_samples),\n                                                0.00, 0.99) +\n            # Add some blur to better show the clipped ones\n            0.01*np.random.random_sample(size=n_samples) )","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.828213Z","iopub.execute_input":"2023-09-22T19:32:40.828512Z","iopub.status.idle":"2023-09-22T19:32:40.840509Z","shell.execute_reply.started":"2023-09-22T19:32:40.828482Z","shell.execute_reply":"2023-09-22T19:32:40.839736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Show the Confusion Dots plot for a threshold\nthresh = 0.70\n\nconfusion_dots(y_true, y_pred, y_thresh=thresh)\n\n# Calcualte the Precision and Recall\nnTN = sum( (y_true == 0) & (y_pred < thresh))\nnFP = sum( (y_true == 0) & (y_pred > thresh))\nnFN = sum( (y_true == 1) & (y_pred < thresh))\nnTP = sum( (y_true == 1) & (y_pred > thresh))\n\nprint(\"nTN = \", nTN, \"nFP = \", nFP)\nprint(\"nFN = \", nFN, \"nTP = \", nTP, \"\\n\")\n\n# Precision and Recall\nprint(\"Precision = {:.3f}\".format(nTP/(nTP+nFP)))\nprint(\"Recall = {:.3f}\".format(nTP/(nTP+nFN)))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:40.841476Z","iopub.execute_input":"2023-09-22T19:32:40.841822Z","iopub.status.idle":"2023-09-22T19:32:41.172273Z","shell.execute_reply.started":"2023-09-22T19:32:40.841792Z","shell.execute_reply":"2023-09-22T19:32:41.171204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the Precision and Recall for a range of thresholds.\nthreshes = [0.995, 0.95, 0.9, 0.8, 0.7, 0.6, 0.5,\n            0.4, 0.3, 0.2, 0.1, 0.05, 0.00]\nRecalls = []\nPrecs = []\nfor thresh in threshes:\n    nTN = sum( (y_true == 0) & (y_pred < thresh))\n    nFP = sum( (y_true == 0) & (y_pred > thresh))\n    nFN = sum( (y_true == 1) & (y_pred < thresh))\n    nTP = sum( (y_true == 1) & (y_pred > thresh))\n    Recalls = Recalls + [nTP/(nTP+nFN)]\n    Precs = Precs + [nTP/(nTP+nFP)]","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:41.174081Z","iopub.execute_input":"2023-09-22T19:32:41.174830Z","iopub.status.idle":"2023-09-22T19:32:41.191567Z","shell.execute_reply.started":"2023-09-22T19:32:41.174788Z","shell.execute_reply":"2023-09-22T19:32:41.190220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the Precision vs Recall\nplt.figure(figsize=(5,5))\nplt.plot(Recalls, Precs, c='gray')\nplt.scatter(Recalls, Precs, s=80, c='blue', marker='.')\nplt.xlabel(\"Recall\"); plt.xlim(0,1.05)\nplt.ylabel(\"Precision\"); plt.ylim(0,1.05)\nplt.title('A Precision vs Recall curve')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:41.195724Z","iopub.execute_input":"2023-09-22T19:32:41.196045Z","iopub.status.idle":"2023-09-22T19:32:41.480151Z","shell.execute_reply.started":"2023-09-22T19:32:41.196017Z","shell.execute_reply":"2023-09-22T19:32:41.479292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Calculate the Average Precision using sklearn formula.\nAP_sum = 0.0\nfor ipt in range(1,len(Recalls)):\n    AP_sum += (Recalls[ipt] - Recalls[ipt-1]) * Precs[ipt]\n    \nprint(\"\\nThe Average Precision is {:.6f}\".format(AP_sum))","metadata":{"execution":{"iopub.status.busy":"2023-09-22T19:32:41.481327Z","iopub.execute_input":"2023-09-22T19:32:41.481858Z","iopub.status.idle":"2023-09-22T19:32:41.488390Z","shell.execute_reply.started":"2023-09-22T19:32:41.481829Z","shell.execute_reply":"2023-09-22T19:32:41.487239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}