{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7003741,"sourceType":"datasetVersion","datasetId":4026344}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Modification of Competition AP metric calculation","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\ndef match_series(pred_locs: np.ndarray, pred_probas: np.ndarray, gt_locs, tolerance=12 * 30):\n    \"\"\"\n    Probably faster algorithm for matching, since the gt are disjoint (within tolerance)\n\n    pred_locs: predicted locations of events, assume sorted in ascending order\n    pred_probas: predicted probabilities of events\n    gt_locs: ground truth locations of events (either list[int] or np.ndarray or int32 type)\n    \"\"\"\n    assert pred_locs.shape == pred_probas.shape, \"pred_locs {} and pred_probas {} must have the same shape\".format(pred_locs.shape, pred_probas.shape)\n    assert len(pred_locs.shape) == 1, \"pred_locs {} and pred_probas {} must be 1D\".format(pred_locs.shape, pred_probas.shape)\n    matches = np.zeros_like(pred_locs, dtype=bool)\n\n    if isinstance(gt_locs, list):\n        gt_locs = np.array(gt_locs, dtype=np.int32)\n    else:\n        assert isinstance(gt_locs, np.ndarray), \"gt_locs must be list or np.ndarray\"\n        assert gt_locs.dtype == np.int32, \"gt_locs must be int32 type\"\n\n    # lie within (event_loc - tolerance, event_loc + tolerance), where event_loc in gt_locs\n    idx_lows = np.searchsorted(pred_locs, gt_locs - tolerance + 1)\n    idx_highs = np.searchsorted(pred_locs, gt_locs + tolerance)\n    for k in range(len(gt_locs)):\n        idx_low, idx_high = idx_lows[k], idx_highs[k]\n        if idx_low == idx_high:\n            continue\n        # find argmax within range\n        max_location = idx_low + np.argmax(pred_probas[idx_low:idx_high])\n        matches[max_location] = True\n    return matches\n\nclass EventMetrics:\n    def __init__(self, name:str, tolerance=12 * 30):\n        self.name = name\n        self.tolerance = tolerance\n\n        self.matches = []\n        self.probas = []\n        self.num_positive = 0\n\n    def add(self, pred_locs: np.ndarray, pred_probas: np.ndarray, gt_locs):\n        matches = match_series(pred_locs, pred_probas, gt_locs, tolerance=self.tolerance)\n        self.matches.append(matches)\n        self.probas.append(pred_probas)\n        self.num_positive += len(gt_locs)\n\n    def get(self):\n        matches = np.concatenate(self.matches, axis=0)\n        probas = np.concatenate(self.probas, axis=0)\n\n        # sort by probas in descending order\n        idxs = np.argsort(probas)[::-1]\n        matches = matches[idxs]\n        probas = probas[idxs]\n\n        # compute precision and recall curve (using Kaggle code)\n        distinct_value_indices = np.where(np.diff(probas))[0]\n        threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n        probas = probas[threshold_idxs]\n\n        # Matches become TPs and non-matches FPs as confidence threshold decreases\n        tps = np.cumsum(matches)[threshold_idxs]\n        fps = np.cumsum(~matches)[threshold_idxs]\n\n        precision = tps / (tps + fps)\n        precision[np.isnan(precision)] = 0\n        recall = tps / self.num_positive  # total number of ground truths might be different than total number of matches\n\n        # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n        last_ind = tps.searchsorted(tps[-1])\n        sl = slice(last_ind, None, -1)\n\n        # Final precision is 1 and final recall is 0 and final proba is 1\n        precision, recall, probas = np.r_[precision[sl], 1], np.r_[recall[sl], 0], np.r_[probas[sl], 1]\n\n        # compute average precision\n        average_precision = -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\n        return precision, recall, average_precision, probas\n\n    def write_to_dict(self, x: dict):\n        x[self.name] = self.get()\n\n    def reset(self):\n        self.matches.clear()\n        self.probas.clear()\n        self.num_positive = 0","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:51:52.298496Z","iopub.execute_input":"2023-11-19T15:51:52.298896Z","iopub.status.idle":"2023-11-19T15:51:52.339764Z","shell.execute_reply.started":"2023-11-19T15:51:52.298853Z","shell.execute_reply":"2023-11-19T15:51:52.339032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Original AP computation code\nSource: https://www.kaggle.com/code/metric/event-detection-ap/notebook","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom typing import Dict, List, Tuple\n\n\nclass ParticipantVisibleError(Exception):\n    pass\n\n\n# Set some placeholders for global parameters\nseries_id_column_name = None\ntime_column_name = None\nevent_column_name = None\nscore_column_name = None\nuse_scoring_intervals = None\n\n\ndef score(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n        series_id_column_name: str,\n        time_column_name: str,\n        event_column_name: str,\n        score_column_name: str,\n        use_scoring_intervals: bool = False,\n) -> float:\n    \"\"\"Event Detection Average Precision, an AUCPR metric for event detection in\n    time series and video.\n\n    This metric is similar to IOU-threshold average precision metrics commonly\n    used in object detection. For events occuring in time series, we replace the\n    IOU threshold with a time tolerance.\n\n    Submissions are evaluated on the average precision of detected events,\n    averaged over timestamp error tolerance thresholds, averaged over event\n    classes.\n\n    Detections are matched to ground-truth events within error tolerances, with\n    ambiguities resolved in order of decreasing confidence.\n\n    Detailed Description\n    --------------------\n    Evaluation proceeds in four steps:\n\n    1. Selection - (optional) Predictions not within a series' scoring\n    intervals are dropped.\n    2. Assignment - Predicted events are matched with ground-truth events.\n    3. Scoring - Each group of predictions is scored against its corresponding\n    group of ground-truth events via Average Precision.\n    4. Reduction - The multiple AP scores are averaged to produce a single\n    overall score.\n\n    Selection\n\n    With each series there may be a defined set of scoring intervals giving the\n    intervals of time over which zero or more ground-truth events might be\n    annotated in that series. A prediction will be evaluated only if it falls\n    within a scoring interval. These scoring intervals can be chosen to improve\n    the fairness of evaluation by, for instance, ignoring edge-cases or\n    ambiguous events.\n\n    It is recommended that, if used, scoring intervals be provided for training\n    data but not test data.\n\n    Assignment\n\n    For each set of predictions and ground-truths within the same `event x\n    tolerance x series_id` group, we match each ground-truth to the\n    highest-confidence unmatched prediction occurring within the allowed\n    tolerance.\n\n    Some ground-truths may not be matched to a prediction and some predictions\n    may not be matched to a ground-truth. They will still be accounted for in\n    the scoring, however.\n\n    Scoring\n\n    Collecting the events within each `series_id`, we compute an Average\n    Precision score for each `event x tolerance` group. The average precision\n    score is the area under the (step-wise) precision-recall curve generated by\n    decreasing confidence score thresholds over the predictions. In this\n    calculation, matched predictions over the threshold are scored as TP and\n    unmatched predictions as FP. Unmatched ground-truths are scored as FN.\n\n    Reduction\n\n    The final score is the average of the above AP scores, first averaged over\n    tolerance, then over event.\n\n    Parameters\n    ----------\n    solution : pd.DataFrame, with columns:\n\n        `series_id_column_name` identifier for each time series\n\n        `time_column_name` the time of occurence for each event as a numeric type\n\n        `event_column_name` class label for each event\n\n        The solution contains the time of occurence of one or more types of\n        event within one or more time series. The metric expects the solution to\n        contain the same event types as those given in `tolerances`.\n\n        When `use_scoring_intervals == True`, you may include `start` and `end`\n        events to delimit intervals within which detections will be scored.\n        Detected events (from the user submission) outside of these events will\n        be ignored.\n\n    submission : pd.DataFrame, with columns as above and in addition:\n\n        `score_column_name` the predicted confidence score for the detected event\n\n    tolerances : Dict[str, List[float]]\n\n        Maps each event class to a list of timestamp tolerances used\n        for matching detections to ground-truth events.\n\n    use_scoring_intervals: bool, default False\n\n        Whether to ignore predicted events outside intervals delimited\n        by `'start'` and `'end'` events in the solution. When `False`,\n        the solution should not include `'start'` and `'end'` events.\n        See the examples for illustration.\n\n    Returns\n    -------\n    event_detection_ap : float\n        The mean average precision of the detected events.\n    \"\"\"\n    # Validate metric parameters\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{column_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)\n\n\ndef filter_detections(\n        detections: pd.DataFrame, intervals: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Drop detections not inside a scoring interval.\"\"\"\n    detection_time = detections.loc[:, time_column_name].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.\n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)\n\n\ndef match_detections(\n        tolerance: float, ground_truths: pd.DataFrame, detections: pd.DataFrame\n) -> pd.DataFrame:\n    \"\"\"Match detections to ground truth events. Arguments are taken from a common event x tolerance x series_id evaluation group.\"\"\"\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted\n\n\ndef precision_recall_curve(\n        matches: np.ndarray, scores: np.ndarray, p: int\n) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # total number of ground truths might be different than total number of matches\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]\n\n\ndef average_precision_score(matches: np.ndarray, scores: np.ndarray, p: int) -> float:\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])\n\n\ndef event_detection_ap(\n        solution: pd.DataFrame,\n        submission: pd.DataFrame,\n        tolerances: Dict[str, List[float]],\n) -> float:\n\n    # Ensure solution and submission are sorted properly\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation)\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:51:52.484904Z","iopub.execute_input":"2023-11-19T15:51:52.485519Z","iopub.status.idle":"2023-11-19T15:51:52.810257Z","shell.execute_reply.started":"2023-11-19T15:51:52.485487Z","shell.execute_reply":"2023-11-19T15:51:52.808940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Testing","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport time\n\ntolerances = [1, 3, 5, 7.5, 10, 12.5, 15, 20, 25, 30]\ntolerances = [tolerance * 12 for tolerance in tolerances]\n\nsubmission = pd.read_csv(\"/kaggle/input/kagglechildsleep-dummy-submission/dummy_submission.csv\")\nsolution = pd.read_csv(\"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\")\nsolution = solution.dropna()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:55:28.780593Z","iopub.execute_input":"2023-11-19T15:55:28.780930Z","iopub.status.idle":"2023-11-19T15:55:29.059443Z","shell.execute_reply.started":"2023-11-19T15:55:28.780904Z","shell.execute_reply":"2023-11-19T15:55:29.058762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Faster Version","metadata":{}},{"cell_type":"code","source":"all_series_ids = list(submission[\"series_id\"].unique())\nall_series_ids = np.array(all_series_ids, dtype=\"object\")\n\nevents = solution\nper_seriesid_events = {}\nfor series_id in all_series_ids:\n    per_seriesid_events[series_id] = {\n        \"onset\": [], \"wakeup\": []\n    }\n    series_events = events.loc[events[\"series_id\"] == series_id]\n    onsets = series_events.loc[series_events[\"event\"] == \"onset\"][\"step\"]\n    wakeups = series_events.loc[series_events[\"event\"] == \"wakeup\"][\"step\"]\n    if len(onsets) > 0:\n        per_seriesid_events[series_id][\"onset\"].extend(onsets.to_numpy(np.int32))\n    if len(wakeups) > 0:\n        per_seriesid_events[series_id][\"wakeup\"].extend(wakeups.to_numpy(np.int32))\n\ngt_events = per_seriesid_events","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:55:40.354171Z","iopub.execute_input":"2023-11-19T15:55:40.354485Z","iopub.status.idle":"2023-11-19T15:55:40.854710Z","shell.execute_reply.started":"2023-11-19T15:55:40.354459Z","shell.execute_reply":"2023-11-19T15:55:40.853282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ctime = time.time()\n\npredicted_events = {}\nfor series_id in submission[\"series_id\"].unique():\n    series_submission = submission.loc[submission[\"series_id\"] == series_id]\n    series_onset = series_submission.loc[series_submission[\"event\"] == \"onset\"]\n    series_wakeup = series_submission.loc[series_submission[\"event\"] == \"wakeup\"]\n    predicted_events[series_id] = {\n        \"onset\": series_onset[\"step\"].to_numpy(dtype=np.int32),\n        \"onset_proba\": series_onset[\"score\"].to_numpy(dtype=np.float32),\n        \"wakeup\": series_wakeup[\"step\"].to_numpy(dtype=np.int32),\n        \"wakeup_proba\": series_wakeup[\"score\"].to_numpy(dtype=np.float32)\n    }\n\n\nap_onset_metrics = [EventMetrics(name=\"\", tolerance=tolerance) for tolerance in tolerances]\nap_wakeup_metrics = [EventMetrics(name=\"\", tolerance=tolerance) for tolerance in tolerances]\n\nfor series_id in all_series_ids:\n    # get the ground truth\n    gt_onset_locs = gt_events[series_id][\"onset\"]\n    gt_wakeup_locs = gt_events[series_id][\"wakeup\"]\n\n    # get the predictions\n    preds_onset = predicted_events[series_id][\"onset\"]\n    preds_wakeup = predicted_events[series_id][\"wakeup\"]\n    onset_IOU_probas = predicted_events[series_id][\"onset_proba\"]\n    wakeup_IOU_probas = predicted_events[series_id][\"wakeup_proba\"]\n\n    # add info\n    for ap_onset_metric, ap_wakeup_metric in zip(ap_onset_metrics, ap_wakeup_metrics):\n        ap_onset_metric.add(pred_locs=preds_onset, pred_probas=onset_IOU_probas, gt_locs=gt_onset_locs)\n        ap_wakeup_metric.add(pred_locs=preds_wakeup, pred_probas=wakeup_IOU_probas, gt_locs=gt_wakeup_locs)\n\n# compute average precision\nap_onset_precisions, ap_onset_recalls, ap_onset_average_precisions, ap_onset_probas = [], [], [], []\nap_wakeup_precisions, ap_wakeup_recalls, ap_wakeup_average_precisions, ap_wakeup_probas = [], [], [], []\nfor ap_onset_metric, ap_wakeup_metric in zip(ap_onset_metrics, ap_wakeup_metrics):\n    ap_onset_precision, ap_onset_recall, ap_onset_average_precision, ap_onset_proba = ap_onset_metric.get()\n    ap_wakeup_precision, ap_wakeup_recall, ap_wakeup_average_precision, ap_wakeup_proba = ap_wakeup_metric.get()\n    ap_onset_precisions.append(ap_onset_precision)\n    ap_onset_recalls.append(ap_onset_recall)\n    ap_onset_average_precisions.append(ap_onset_average_precision)\n    ap_onset_probas.append(ap_onset_proba)\n    ap_wakeup_precisions.append(ap_wakeup_precision)\n    ap_wakeup_recalls.append(ap_wakeup_recall)\n    ap_wakeup_average_precisions.append(ap_wakeup_average_precision)\n    ap_wakeup_probas.append(ap_wakeup_proba)\n\n# print the mean of average precisions\nprint(\"Time Elapsed: {}\".format(time.time() - ctime))\nprint(\"Onset Average Precision: {}\".format(np.mean(ap_onset_average_precisions)))\nprint(\"Wakeup Average Precision: {}\".format(np.mean(ap_wakeup_average_precisions)))\nprint(\"Combined Average Precision: {}\".format((np.mean(ap_onset_average_precisions) + np.mean(ap_wakeup_average_precisions)) / 2))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:56:22.094660Z","iopub.execute_input":"2023-11-19T15:56:22.094981Z","iopub.status.idle":"2023-11-19T15:56:31.403925Z","shell.execute_reply.started":"2023-11-19T15:56:22.094955Z","shell.execute_reply":"2023-11-19T15:56:31.402403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Kaggle AP metric","metadata":{}},{"cell_type":"code","source":"ctime = time.time()\ncomputed_score = score(solution, submission,\n                                  tolerances={\"onset\": tolerances,\n                                              \"wakeup\": tolerances},\n                                  series_id_column_name=\"series_id\",\n                                  time_column_name=\"step\",\n                                  event_column_name=\"event\",\n                                  score_column_name=\"score\",\n                                  use_scoring_intervals=False)\nprint(\"Evaluation Time: {}\".format(time.time() - ctime))\n\nprint(\"Score: {}\".format(computed_score))","metadata":{"execution":{"iopub.status.busy":"2023-11-19T15:56:55.891289Z","iopub.execute_input":"2023-11-19T15:56:55.891633Z","iopub.status.idle":"2023-11-19T16:03:00.958222Z","shell.execute_reply.started":"2023-11-19T15:56:55.891592Z","shell.execute_reply":"2023-11-19T16:03:00.956669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot Diagnosis","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.title(\"Onset {}\".format(tolerances[-1]))\nplt.xlabel(\"Recall\")\nplt.ylabel(\"Precision\")\nplt.xlim(0, 1)\nplt.ylim(0, 1)\nplt.plot(ap_onset_recalls[-1], ap_onset_precisions[-1])\nplt.text(0.5, 0.5, \"AP: {:.4f}\".format(ap_onset_average_precisions[-1]), horizontalalignment=\"center\", verticalalignment=\"center\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-19T16:04:49.498398Z","iopub.execute_input":"2023-11-19T16:04:49.498788Z","iopub.status.idle":"2023-11-19T16:04:49.663911Z","shell.execute_reply.started":"2023-11-19T16:04:49.498755Z","shell.execute_reply":"2023-11-19T16:04:49.662991Z"},"trusted":true},"execution_count":null,"outputs":[]}]}