{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":151018325,"sourceType":"kernelVersion"}],"dockerImageVersionId":30559,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Installing torchvision nightly & pycocotools","metadata":{}},{"cell_type":"code","source":"!pip show torchvision","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:03:49.962555Z","iopub.execute_input":"2023-11-17T08:03:49.962925Z","iopub.status.idle":"2023-11-17T08:04:01.717671Z","shell.execute_reply.started":"2023-11-17T08:03:49.962892Z","shell.execute_reply":"2023-11-17T08:04:01.716681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\n\nprint(torch.version.cuda) # '11.8' for P100","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:04:01.719796Z","iopub.execute_input":"2023-11-17T08:04:01.720159Z","iopub.status.idle":"2023-11-17T08:04:04.680600Z","shell.execute_reply.started":"2023-11-17T08:04:01.720123Z","shell.execute_reply":"2023-11-17T08:04:04.679639Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!yes y | pip uninstall torch torchvision\n!yes y | pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu118","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:04:04.682050Z","iopub.execute_input":"2023-11-17T08:04:04.682536Z","iopub.status.idle":"2023-11-17T08:06:22.402802Z","shell.execute_reply.started":"2023-11-17T08:04:04.682499Z","shell.execute_reply":"2023-11-17T08:06:22.401682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Restart kernel after installation:\nimport IPython\n\nIPython.Application.instance().kernel.do_shutdown(True)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:06:24.861795Z","iopub.execute_input":"2023-11-17T08:06:24.862220Z","iopub.status.idle":"2023-11-17T08:06:24.871662Z","shell.execute_reply.started":"2023-11-17T08:06:24.862175Z","shell.execute_reply":"2023-11-17T08:06:24.870570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install pycocotools","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:06:37.076219Z","iopub.execute_input":"2023-11-17T08:06:37.076621Z","iopub.status.idle":"2023-11-17T08:06:49.213875Z","shell.execute_reply.started":"2023-11-17T08:06:37.076585Z","shell.execute_reply":"2023-11-17T08:06:49.212702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\nimport random\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport torch\nimport torchvision\nfrom torchvision.io import read_image\nfrom torchvision import tv_tensors\nfrom torchvision.transforms import v2 as T\nfrom torchvision.transforms.v2 import functional as F\nfrom torchvision.models.detection.faster_rcnn import FastRCNNPredictor\nfrom torchvision.utils import draw_bounding_boxes\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:02.571236Z","iopub.execute_input":"2023-11-17T08:07:02.571999Z","iopub.status.idle":"2023-11-17T08:07:07.137004Z","shell.execute_reply.started":"2023-11-17T08:07:02.571965Z","shell.execute_reply":"2023-11-17T08:07:07.136180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = Path(\"/kaggle/input/child-mind-institute-detect-sleep-states\")\ntrain_events = pd.read_csv(data_folder/\"train_events.csv\")\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:07.138435Z","iopub.execute_input":"2023-11-17T08:07:07.139013Z","iopub.status.idle":"2023-11-17T08:07:07.194433Z","shell.execute_reply.started":"2023-11-17T08:07:07.138985Z","shell.execute_reply":"2023-11-17T08:07:07.193476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_events['series_id'].unique()\nlen(series_ids)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:10.052684Z","iopub.execute_input":"2023-11-17T08:07:10.053062Z","iopub.status.idle":"2023-11-17T08:07:10.064505Z","shell.execute_reply.started":"2023-11-17T08:07:10.053033Z","shell.execute_reply":"2023-11-17T08:07:10.063402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_prep_folder = Path(\"/kaggle/input/d-s-s-faster-r-cnn-data-prep\")\nimages_folder = data_prep_folder/\"images\"","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:19.259345Z","iopub.execute_input":"2023-11-17T08:07:19.259729Z","iopub.status.idle":"2023-11-17T08:07:19.264334Z","shell.execute_reply.started":"2023-11-17T08:07:19.259700Z","shell.execute_reply":"2023-11-17T08:07:19.263373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_names = []\nfor file_path in images_folder.glob(\"*.jpg\"):\n    file_name = str(file_path).split(\"/\")[-1]\n    image_names.append(file_name)\nlen(image_names)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:19.871704Z","iopub.execute_input":"2023-11-17T08:07:19.872509Z","iopub.status.idle":"2023-11-17T08:07:20.137630Z","shell.execute_reply.started":"2023-11-17T08:07:19.872469Z","shell.execute_reply":"2023-11-17T08:07:20.136718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_names[:5]","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:23.927880Z","iopub.execute_input":"2023-11-17T08:07:23.928269Z","iopub.status.idle":"2023-11-17T08:07:23.934475Z","shell.execute_reply.started":"2023-11-17T08:07:23.928241Z","shell.execute_reply":"2023-11-17T08:07:23.933546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotations_df = pd.read_csv(data_prep_folder/\"annotations_8_30pm_utc_cutoff.csv\")\nannotations_df","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:25.445587Z","iopub.execute_input":"2023-11-17T08:07:25.445969Z","iopub.status.idle":"2023-11-17T08:07:25.492606Z","shell.execute_reply.started":"2023-11-17T08:07:25.445940Z","shell.execute_reply":"2023-11-17T08:07:25.491696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(annotations_df['series_id'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:29.309093Z","iopub.execute_input":"2023-11-17T08:07:29.309443Z","iopub.status.idle":"2023-11-17T08:07:29.317088Z","shell.execute_reply.started":"2023-11-17T08:07:29.309416Z","shell.execute_reply":"2023-11-17T08:07:29.316131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We see that the number of series IDs in `annotations_df` is less than 277. This is because there are a few series IDs with no sleep events.","metadata":{}},{"cell_type":"code","source":"len(annotations_df['image_name'].unique())","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:36.459476Z","iopub.execute_input":"2023-11-17T08:07:36.460114Z","iopub.status.idle":"2023-11-17T08:07:36.467338Z","shell.execute_reply.started":"2023-11-17T08:07:36.460085Z","shell.execute_reply":"2023-11-17T08:07:36.466418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Moreover, we see that the number of image names in `annotations_df` is less than 7,696. This is because there are many images with no sleep events.","metadata":{}},{"cell_type":"code","source":"window_properties_df = pd.read_csv(data_prep_folder/\"window_properties_8_30pm_utc_cutoff.csv\")\nwindow_properties_df","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:07:55.325211Z","iopub.execute_input":"2023-11-17T08:07:55.325578Z","iopub.status.idle":"2023-11-17T08:07:55.362506Z","shell.execute_reply.started":"2023-11-17T08:07:55.325550Z","shell.execute_reply":"2023-11-17T08:07:55.361539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model validation scheme","metadata":{}},{"cell_type":"markdown","source":"Let's create a holdout validation scheme with 20% of the series IDs in the validation set.","metadata":{}},{"cell_type":"code","source":"num_val_series_ids = round(0.2 * len(series_ids))\nnum_val_series_ids","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:01.115042Z","iopub.execute_input":"2023-11-17T08:08:01.116110Z","iopub.status.idle":"2023-11-17T08:08:01.121989Z","shell.execute_reply.started":"2023-11-17T08:08:01.116078Z","shell.execute_reply":"2023-11-17T08:08:01.121017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Our validation set will contain 55 series IDs. Let's select them randomly.","metadata":{}},{"cell_type":"code","source":"np.random.seed(42)\nseries_ids_in_val = np.random.choice(series_ids, size=num_val_series_ids, replace=False)\nseries_ids_in_val","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:07.907670Z","iopub.execute_input":"2023-11-17T08:08:07.908498Z","iopub.status.idle":"2023-11-17T08:08:07.915571Z","shell.execute_reply.started":"2023-11-17T08:08:07.908467Z","shell.execute_reply":"2023-11-17T08:08:07.914610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(series_ids_in_val)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:12.956753Z","iopub.execute_input":"2023-11-17T08:08:12.957127Z","iopub.status.idle":"2023-11-17T08:08:12.963667Z","shell.execute_reply.started":"2023-11-17T08:08:12.957099Z","shell.execute_reply":"2023-11-17T08:08:12.962720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids_in_train = series_ids[~ np.isin(series_ids, series_ids_in_val)]\nseries_ids_in_train","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:15.671151Z","iopub.execute_input":"2023-11-17T08:08:15.671991Z","iopub.status.idle":"2023-11-17T08:08:15.680277Z","shell.execute_reply.started":"2023-11-17T08:08:15.671962Z","shell.execute_reply":"2023-11-17T08:08:15.679298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(series_ids_in_train)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:16.070079Z","iopub.execute_input":"2023-11-17T08:08:16.070423Z","iopub.status.idle":"2023-11-17T08:08:16.076750Z","shell.execute_reply.started":"2023-11-17T08:08:16.070397Z","shell.execute_reply":"2023-11-17T08:08:16.075722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(series_ids_in_train) + len(series_ids_in_val)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:18.788038Z","iopub.execute_input":"2023-11-17T08:08:18.788406Z","iopub.status.idle":"2023-11-17T08:08:18.794603Z","shell.execute_reply.started":"2023-11-17T08:08:18.788378Z","shell.execute_reply":"2023-11-17T08:08:18.793579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, we need to go from series IDs to image names.\n\nWe've already seen that we cannot rely on `annotations_df` because it doesn't contain all the image names. Hence, we'll rely on `window_properties_df` to create our training and validation sets.","metadata":{}},{"cell_type":"code","source":"train = window_properties_df.loc[window_properties_df['series_id'].isin(series_ids_in_train)].reset_index(drop=True)\nval = window_properties_df.loc[window_properties_df['series_id'].isin(series_ids_in_val)].reset_index(drop=True)\nlen(train), len(val)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:29.001556Z","iopub.execute_input":"2023-11-17T08:08:29.001957Z","iopub.status.idle":"2023-11-17T08:08:29.017046Z","shell.execute_reply.started":"2023-11-17T08:08:29.001926Z","shell.execute_reply":"2023-11-17T08:08:29.015978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train)  + len(val)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:08:29.930020Z","iopub.execute_input":"2023-11-17T08:08:29.930367Z","iopub.status.idle":"2023-11-17T08:08:29.936205Z","shell.execute_reply.started":"2023-11-17T08:08:29.930341Z","shell.execute_reply":"2023-11-17T08:08:29.935297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"To compute the competition metric, we'll need to (a) split the `train_events` data frame into a validation subset and a training subset, and (b) drop rows containing missing values for `'step'` & `'timestamp'`.","metadata":{}},{"cell_type":"code","source":"val_events = train_events.loc[train_events['series_id'].isin(series_ids_in_val)]\nval_events = val_events.loc[val_events['timestamp'].notna()].reset_index(drop=True)\nval_events","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:26.967853Z","iopub.execute_input":"2023-11-17T08:09:26.968593Z","iopub.status.idle":"2023-11-17T08:09:26.988456Z","shell.execute_reply.started":"2023-11-17T08:09:26.968564Z","shell.execute_reply":"2023-11-17T08:09:26.987528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events.loc[train_events['series_id'].isin(series_ids_in_train)]\ntrain_events = train_events.loc[train_events['timestamp'].notna()].reset_index(drop=True)\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:31.028260Z","iopub.execute_input":"2023-11-17T08:09:31.029098Z","iopub.status.idle":"2023-11-17T08:09:31.049646Z","shell.execute_reply.started":"2023-11-17T08:09:31.029068Z","shell.execute_reply":"2023-11-17T08:09:31.048702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Code for computing the competition metric","metadata":{}},{"cell_type":"code","source":"class ParticipantVisibleError(Exception):\n    pass","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:36.570529Z","iopub.execute_input":"2023-11-17T08:09:36.571418Z","iopub.status.idle":"2023-11-17T08:09:36.575490Z","shell.execute_reply.started":"2023-11-17T08:09:36.571388Z","shell.execute_reply":"2023-11-17T08:09:36.574528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_id_column_name = None\ntime_column_name = None\nevent_column_name = None\nscore_column_name = None\nuse_scoring_intervals = None","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:37.042517Z","iopub.execute_input":"2023-11-17T08:09:37.043224Z","iopub.status.idle":"2023-11-17T08:09:37.047764Z","shell.execute_reply.started":"2023-11-17T08:09:37.043192Z","shell.execute_reply":"2023-11-17T08:09:37.046716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def score(solution, submission, tolerances, series_id_column_name, time_column_name, event_column_name, score_column_name, use_scoring_intervals=False):\n    # Validate metric parameters.\n    assert len(tolerances) > 0, \"Events must have defined tolerances.\"\n    assert set(tolerances.keys()) == set(solution[event_column_name]).difference({'start', 'end'}),\\\n        (f\"Solution column {event_column_name} must contain the same events \"\n         \"as defined in tolerances.\")\n    assert pd.api.types.is_numeric_dtype(solution[time_column_name]),\\\n        f\"Solution column {time_column_name} must be of numeric type.\"\n\n    # Validate submission format.\n    for column_name in [\n        series_id_column_name,\n        time_column_name,\n        event_column_name,\n        score_column_name,\n    ]:\n        if column_name not in submission.columns:\n            raise ParticipantVisibleError(f\"Submission must have column '{column_name}'.\")\n\n    if not pd.api.types.is_numeric_dtype(submission[time_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{time_column_name}' must be of numeric type.\"\n        )\n    if not pd.api.types.is_numeric_dtype(submission[score_column_name]):\n        raise ParticipantVisibleError(\n            f\"Submission column '{score_column_name}' must be of numeric type.\"\n        )\n\n    # Set these globally to avoid passing around a bunch of arguments.\n    globals()['series_id_column_name'] = series_id_column_name\n    globals()['time_column_name'] = time_column_name\n    globals()['event_column_name'] = event_column_name\n    globals()['score_column_name'] = score_column_name\n    globals()['use_scoring_intervals'] = use_scoring_intervals\n\n    return event_detection_ap(solution, submission, tolerances)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:37.566059Z","iopub.execute_input":"2023-11-17T08:09:37.566818Z","iopub.status.idle":"2023-11-17T08:09:37.575922Z","shell.execute_reply.started":"2023-11-17T08:09:37.566780Z","shell.execute_reply":"2023-11-17T08:09:37.574829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def filter_detections(detections, intervals):\n    detection_time = detections.loc[:, time_column_name].sort_values().to_numpy()\n    intervals = intervals.to_numpy()\n    is_scored = np.full_like(detection_time, False, dtype=bool)\n\n    i, j = 0, 0\n    while i < len(detection_time) and j < len(intervals):\n        time = detection_time[i]\n        int_ = intervals[j]\n\n        # If the detection is prior in time to the interval, go to the next detection.\n        if time < int_.left:\n            i += 1\n        # If the detection is inside the interval, keep it and go to the next detection.\n        elif time in int_:\n            is_scored[i] = True\n            i += 1\n        # If the detection is later in time, go to the next interval.\n        else:\n            j += 1\n\n    return detections.loc[is_scored].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:38.832355Z","iopub.execute_input":"2023-11-17T08:09:38.832830Z","iopub.status.idle":"2023-11-17T08:09:38.841226Z","shell.execute_reply.started":"2023-11-17T08:09:38.832787Z","shell.execute_reply":"2023-11-17T08:09:38.840186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def match_detections(tolerance, ground_truths, detections):\n    detections_sorted = detections.sort_values(score_column_name, ascending=False).dropna()\n    is_matched = np.full_like(detections_sorted[event_column_name], False, dtype=bool)\n    gts_matched = set()\n    for i, det in enumerate(detections_sorted.itertuples(index=False)):\n        best_error = tolerance\n        best_gt = None\n\n        for gt in ground_truths.itertuples(index=False):\n            error = abs(getattr(det, time_column_name) - getattr(gt, time_column_name))\n            if error < best_error and gt not in gts_matched:\n                best_gt = gt\n                best_error = error\n\n        if best_gt is not None:\n            is_matched[i] = True\n            gts_matched.add(best_gt)\n\n    detections_sorted['matched'] = is_matched\n\n    return detections_sorted","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:39.322932Z","iopub.execute_input":"2023-11-17T08:09:39.323585Z","iopub.status.idle":"2023-11-17T08:09:39.330857Z","shell.execute_reply.started":"2023-11-17T08:09:39.323554Z","shell.execute_reply":"2023-11-17T08:09:39.329852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def precision_recall_curve(matches, scores, p):\n    if len(matches) == 0:\n        return [1], [0], []\n\n    # Sort matches by decreasing confidence.\n    idxs = np.argsort(scores, kind='stable')[::-1]\n    scores = scores[idxs]\n    matches = matches[idxs]\n\n    distinct_value_indices = np.where(np.diff(scores))[0]\n    threshold_idxs = np.r_[distinct_value_indices, matches.size - 1]\n    thresholds = scores[threshold_idxs]\n\n    # Matches become TPs and non-matches FPs as confidence threshold decreases.\n    tps = np.cumsum(matches)[threshold_idxs]\n    fps = np.cumsum(~matches)[threshold_idxs]\n\n    precision = tps / (tps + fps)\n    precision[np.isnan(precision)] = 0\n    recall = tps / p  # Total number of ground truths might be different than total number of matches.\n\n    # Stop when full recall attained and reverse the outputs so recall is non-increasing.\n    last_ind = tps.searchsorted(tps[-1])\n    sl = slice(last_ind, None, -1)\n\n    # Final precision is 1 and final recall is 0.\n    return np.r_[precision[sl], 1], np.r_[recall[sl], 0], thresholds[sl]","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:39.765855Z","iopub.execute_input":"2023-11-17T08:09:39.766500Z","iopub.status.idle":"2023-11-17T08:09:39.774478Z","shell.execute_reply.started":"2023-11-17T08:09:39.766471Z","shell.execute_reply":"2023-11-17T08:09:39.773581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def average_precision_score(matches, scores, p):\n    precision, recall, _ = precision_recall_curve(matches, scores, p)\n    # Compute step integral.\n    return -np.sum(np.diff(recall) * np.array(precision)[:-1])","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:40.246613Z","iopub.execute_input":"2023-11-17T08:09:40.247025Z","iopub.status.idle":"2023-11-17T08:09:40.252547Z","shell.execute_reply.started":"2023-11-17T08:09:40.246996Z","shell.execute_reply":"2023-11-17T08:09:40.251537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def event_detection_ap(solution, submission, tolerances):\n    # Ensure solution and submission are sorted properly.\n    solution = solution.sort_values([series_id_column_name, time_column_name])\n    submission = submission.sort_values([series_id_column_name, time_column_name])\n\n    # Extract scoring intervals.\n    if use_scoring_intervals:\n        intervals = (\n            solution\n            .query(\"event in ['start', 'end']\")\n            .assign(interval=lambda x: x.groupby([series_id_column_name, event_column_name]).cumcount())\n            .pivot(\n                index='interval',\n                columns=[series_id_column_name, event_column_name],\n                values=time_column_name,\n            )\n            .stack(series_id_column_name)\n            .swaplevel()\n            .sort_index()\n            .loc[:, ['start', 'end']]\n            .apply(lambda x: pd.Interval(*x, closed='both'), axis=1)\n        )\n\n    # Extract ground-truth events.\n    ground_truths = (\n        solution\n        .query(\"event not in ['start', 'end']\")\n        .reset_index(drop=True)\n    )\n\n    # Map each event class to its prevalence (needed for recall calculation).\n    class_counts = ground_truths.value_counts(event_column_name).to_dict()\n\n    # Create table for detections with a column indicating a match to a ground-truth event.\n    detections = submission.assign(matched = False)\n\n    # Remove detections outside of scoring intervals.\n    if use_scoring_intervals:\n        detections_filtered = []\n        for (det_group, dets), (int_group, ints) in zip(\n            detections.groupby(series_id_column_name), intervals.groupby(series_id_column_name)\n        ):\n            assert det_group == int_group\n            detections_filtered.append(filter_detections(dets, ints))\n        detections_filtered = pd.concat(detections_filtered, ignore_index=True)\n    else:\n        detections_filtered = detections\n\n    # Create table of event-class x tolerance x series_id values.\n    aggregation_keys = pd.DataFrame(\n        [(ev, tol, vid)\n         for ev in tolerances.keys()\n         for tol in tolerances[ev]\n         for vid in ground_truths[series_id_column_name].unique()],\n        columns=[event_column_name, 'tolerance', series_id_column_name],\n    )\n\n    # Create match evaluation groups: event-class x tolerance x series_id.\n    detections_grouped = (\n        aggregation_keys\n        .merge(detections_filtered, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    ground_truths_grouped = (\n        aggregation_keys\n        .merge(ground_truths, on=[event_column_name, series_id_column_name], how='left')\n        .groupby([event_column_name, 'tolerance', series_id_column_name])\n    )\n    # Match detections to ground truth events by evaluation group.\n    detections_matched = []\n    for key in aggregation_keys.itertuples(index=False):\n        dets = detections_grouped.get_group(key)\n        gts = ground_truths_grouped.get_group(key)\n        detections_matched.append(\n            match_detections(dets['tolerance'].iloc[0], gts, dets)\n        )\n    detections_matched = pd.concat(detections_matched)\n\n    # Compute AP per event x tolerance group.\n    event_classes = ground_truths[event_column_name].unique()\n    ap_table = (\n        detections_matched\n        .query(\"event in @event_classes\")\n        .groupby([event_column_name, 'tolerance']).apply(\n            lambda group: average_precision_score(\n                group['matched'].to_numpy(),\n                group[score_column_name].to_numpy(),\n                class_counts[group[event_column_name].iat[0]],\n            )\n        )\n    )\n    # Average over tolerances, then over event classes.\n    mean_ap = ap_table.groupby(event_column_name).mean().sum() / len(event_classes)\n\n    return mean_ap","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:41.020723Z","iopub.execute_input":"2023-11-17T08:09:41.021637Z","iopub.status.idle":"2023-11-17T08:09:41.037795Z","shell.execute_reply.started":"2023-11-17T08:09:41.021607Z","shell.execute_reply":"2023-11-17T08:09:41.036811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tolerances = {\n    \"onset\" : [12, 36, 60, 90, 120, 150, 180, 240, 300, 360],\n    'wakeup': [12, 36, 60, 90, 120, 150, 180, 240, 300, 360]    \n}","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:41.479169Z","iopub.execute_input":"2023-11-17T08:09:41.479540Z","iopub.status.idle":"2023-11-17T08:09:41.484702Z","shell.execute_reply.started":"2023-11-17T08:09:41.479511Z","shell.execute_reply":"2023-11-17T08:09:41.483702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"column_names = {\n    'series_id_column_name': 'series_id',\n    'time_column_name': 'step',\n    'event_column_name': 'event',\n    'score_column_name': 'score'\n}","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:41.944231Z","iopub.execute_input":"2023-11-17T08:09:41.945013Z","iopub.status.idle":"2023-11-17T08:09:41.949659Z","shell.execute_reply.started":"2023-11-17T08:09:41.944981Z","shell.execute_reply":"2023-11-17T08:09:41.948607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_transforms():\n    transforms = []\n    transforms.append(T.ToDtype(torch.float, scale=True))\n    transforms.append(T.ToPureTensor())\n    transforms.append(T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))\n    return T.Compose(transforms)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:44.580069Z","iopub.execute_input":"2023-11-17T08:09:44.580802Z","iopub.status.idle":"2023-11-17T08:09:44.586058Z","shell.execute_reply.started":"2023-11-17T08:09:44.580772Z","shell.execute_reply":"2023-11-17T08:09:44.585039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_transforms = get_transforms()\nreverse_label_mapping = {1: \"onset\", 2: \"wakeup\"}","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:09:44.622436Z","iopub.execute_input":"2023-11-17T08:09:44.622808Z","iopub.status.idle":"2023-11-17T08:09:44.627506Z","shell.execute_reply.started":"2023-11-17T08:09:44.622771Z","shell.execute_reply":"2023-11-17T08:09:44.626499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_metric():\n    all_val_preds = []\n    for i, series_id in tqdm(enumerate(series_ids_in_val)): # i is the index of the series.\n        series_rows = window_properties_df.loc[window_properties_df['series_id'] == series_id].reset_index(drop=True)\n        for j in range(len(series_rows)): # j is the index of the image (within the series).\n            image_name = series_rows['image_name'][j]\n            image_path = os.path.join(data_prep_folder, \"images\", image_name)\n            image = read_image(image_path)\n            with torch.no_grad():\n                x = eval_transforms(image)\n                x = x.to(device)\n                predictions = model([x])\n            pred = predictions[0]\n            # Postprocessing: remove 'narrow' boxes as these are false positives.\n            pred_widths = pred['boxes'][:, 2] - pred['boxes'][:, 0]\n            pred['boxes'] = pred['boxes'][pred_widths > 10]\n            pred['labels'] = pred['labels'][pred_widths > 10]\n            pred['scores'] = pred['scores'][pred_widths > 10]\n            if len(pred['labels']) == 0:\n                continue\n            else:\n                pred_x = (pred['boxes'][:, 0] + pred['boxes'][:, 2]) / 2\n                pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n                num_steps_window = series_rows['num_steps_window'][j]\n                if j == 0:\n                    prev_num_steps_cumulative = 0\n                else:\n                    prev_num_steps_cumulative = series_rows['num_steps_cumulative'][j - 1]            \n                for k in range(len(pred_labels)): # k is the index of the bounding box (within the image).\n                    event = {}\n                    step_in_window = (pred_x[k] / 1440) * num_steps_window # All images have width 1,440px.\n                    step_in_series = int(prev_num_steps_cumulative + step_in_window)\n                    event['series_id'] = series_id\n                    event['step'] = step_in_series\n                    event['event'] = pred_labels[k]\n                    event['score'] = pred['scores'][k].item()\n                    all_val_preds.append(event)\n    all_val_preds_df = pd.DataFrame(all_val_preds)\n    all_val_preds_df = all_val_preds_df.sort_values(by=['series_id', 'step']).reset_index(drop=True)\n    all_val_preds_df['row_id'] = np.arange(len(all_val_preds_df))\n    all_val_preds_df = all_val_preds_df[['row_id', 'series_id', 'step', 'event', 'score']]\n    competition_metric = round(score(val_events, all_val_preds_df, tolerances, **column_names), 4)\n    return competition_metric","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:10:42.453356Z","iopub.execute_input":"2023-11-17T08:10:42.454247Z","iopub.status.idle":"2023-11-17T08:10:42.468887Z","shell.execute_reply.started":"2023-11-17T08:10:42.454215Z","shell.execute_reply":"2023-11-17T08:10:42.467892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a custom Dataset class","metadata":{}},{"cell_type":"code","source":"class SleepStatesDataset(torch.utils.data.Dataset):\n    def __init__(self, split_df, annotations_df, images_folder, transforms):\n        self.label_mapping = {'onset': 1, 'wakeup': 2}\n        self.split_df = split_df\n        self.annotations_df = annotations_df\n        self.images_folder = images_folder\n        self.transforms = transforms\n    \n    def __len__(self):\n        return len(self.split_df)\n    \n    def __getitem__(self, idx):\n        image_name = self.split_df['image_name'][idx]\n        image_path = os.path.join(self.images_folder, image_name)\n        image = read_image(image_path)\n        window_df = self.annotations_df.loc[self.annotations_df['image_name'] == image_name]\n        if len(window_df) > 0:            \n            boxes = torch.tensor(window_df.iloc[:, 5:].values, dtype=torch.float32)\n            labels = window_df['label']\n            labels = torch.tensor([self.label_mapping[l] for l in labels], dtype=torch.int64)\n            area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])\n            iscrowd = torch.zeros(size=(len(window_df),), dtype=torch.int64)\n        else:\n            boxes = torch.empty(size=(0, 4), dtype=torch.float32)\n            labels = torch.empty(size=(0,), dtype=torch.int64)\n            area = torch.empty(size=(0,), dtype=torch.float32)\n            iscrowd = torch.empty(size=(0,), dtype=torch.int64)\n        \n        image = tv_tensors.Image(image)\n        target = {}\n        target['image_id'] = image_name\n        target['boxes'] = tv_tensors.BoundingBoxes(boxes, format=\"XYXY\", canvas_size=F.get_size(image))\n        target['labels'] = labels\n        target['area'] = area\n        target['iscrowd'] = iscrowd\n        \n        if self.transforms is not None:\n            image, target = self.transforms(image, target)\n\n        return image, target","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:10:50.967948Z","iopub.execute_input":"2023-11-17T08:10:50.968592Z","iopub.status.idle":"2023-11-17T08:10:50.981074Z","shell.execute_reply.started":"2023-11-17T08:10:50.968564Z","shell.execute_reply":"2023-11-17T08:10:50.980034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's test whether our `SleepStatesDataset` class works.","metadata":{}},{"cell_type":"code","source":"train_dataset = SleepStatesDataset(train, annotations_df, images_folder, get_transforms())\nval_dataset = SleepStatesDataset(val, annotations_df, images_folder, get_transforms())","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:10:53.167319Z","iopub.execute_input":"2023-11-17T08:10:53.168078Z","iopub.status.idle":"2023-11-17T08:10:53.172632Z","shell.execute_reply.started":"2023-11-17T08:10:53.168050Z","shell.execute_reply":"2023-11-17T08:10:53.171711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(train_dataset), len(val_dataset)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:10:56.406393Z","iopub.execute_input":"2023-11-17T08:10:56.406774Z","iopub.status.idle":"2023-11-17T08:10:56.413180Z","shell.execute_reply.started":"2023-11-17T08:10:56.406733Z","shell.execute_reply":"2023-11-17T08:10:56.412246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[0]","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:10:56.476841Z","iopub.execute_input":"2023-11-17T08:10:56.477203Z","iopub.status.idle":"2023-11-17T08:10:56.508822Z","shell.execute_reply.started":"2023-11-17T08:10:56.477176Z","shell.execute_reply":"2023-11-17T08:10:56.507837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Instantiating the model & data loaders","metadata":{}},{"cell_type":"code","source":"def seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nseed_everything(42)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:09.023864Z","iopub.execute_input":"2023-11-17T08:11:09.024243Z","iopub.status.idle":"2023-11-17T08:11:09.032640Z","shell.execute_reply.started":"2023-11-17T08:11:09.024213Z","shell.execute_reply":"2023-11-17T08:11:09.031773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load an object detection model pre-trained on COCO:\nmodel = torchvision.models.detection.fasterrcnn_resnet50_fpn(weights=\"DEFAULT\", max_size=1440)\n# Replace the classifier with a new one, that has num_classes which is user-defined:\nnum_classes = 3 # 2 classes ('onset' & 'wakeup') + the 'background' class\n# Get the number of input features for the box classifier:\nin_features = model.roi_heads.box_predictor.cls_score.in_features\n# Replace the pre-trained box predictor head with a new one:\nmodel.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:20.331593Z","iopub.execute_input":"2023-11-17T08:11:20.332467Z","iopub.status.idle":"2023-11-17T08:11:22.245012Z","shell.execute_reply.started":"2023-11-17T08:11:20.332437Z","shell.execute_reply":"2023-11-17T08:11:22.244165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Download helper functions:\nos.system(\"wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/transforms.py\")\nos.system(\"wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/utils.py\")\nos.system(\"wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/engine.py\")\nos.system(\"wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_utils.py\")\nos.system(\"wget https://raw.githubusercontent.com/pytorch/vision/main/references/detection/coco_eval.py\")","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:24.689213Z","iopub.execute_input":"2023-11-17T08:11:24.689943Z","iopub.status.idle":"2023-11-17T08:11:25.811243Z","shell.execute_reply.started":"2023-11-17T08:11:24.689913Z","shell.execute_reply":"2023-11-17T08:11:25.810318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import utils # utils.py","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:25.813044Z","iopub.execute_input":"2023-11-17T08:11:25.813701Z","iopub.status.idle":"2023-11-17T08:11:25.820151Z","shell.execute_reply.started":"2023-11-17T08:11:25.813665Z","shell.execute_reply":"2023-11-17T08:11:25.819409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=4, collate_fn=utils.collate_fn)\nval_loader = torch.utils.data.DataLoader(val_dataset, batch_size=8, shuffle=False, num_workers=4, collate_fn=utils.collate_fn)\nlen(train_loader), len(val_loader)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:32.600802Z","iopub.execute_input":"2023-11-17T08:11:32.601572Z","iopub.status.idle":"2023-11-17T08:11:32.609375Z","shell.execute_reply.started":"2023-11-17T08:11:32.601542Z","shell.execute_reply":"2023-11-17T08:11:32.608441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's test our model's forward pass on a single batch.","metadata":{}},{"cell_type":"code","source":"images, targets = next(iter(val_loader))\ntype(images), type(targets)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:35.082564Z","iopub.execute_input":"2023-11-17T08:11:35.082979Z","iopub.status.idle":"2023-11-17T08:11:35.951266Z","shell.execute_reply.started":"2023-11-17T08:11:35.082950Z","shell.execute_reply":"2023-11-17T08:11:35.950141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images = [image for image in images]\ntargets = [target for target in targets]\noutput = model(images, targets)\noutput","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:11:38.153173Z","iopub.execute_input":"2023-11-17T08:11:38.154121Z","iopub.status.idle":"2023-11-17T08:11:59.629230Z","shell.execute_reply.started":"2023-11-17T08:11:38.154083Z","shell.execute_reply":"2023-11-17T08:11:59.628099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"from engine import train_one_epoch, evaluate # engine.py","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:12:04.691290Z","iopub.execute_input":"2023-11-17T08:12:04.692197Z","iopub.status.idle":"2023-11-17T08:12:04.715208Z","shell.execute_reply.started":"2023-11-17T08:12:04.692163Z","shell.execute_reply":"2023-11-17T08:12:04.714245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\nmodel.to(device)\nparams = [p for p in model.parameters() if p.requires_grad]\noptimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)\nlr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:12:05.481964Z","iopub.execute_input":"2023-11-17T08:12:05.482352Z","iopub.status.idle":"2023-11-17T08:12:07.814717Z","shell.execute_reply.started":"2023-11-17T08:12:05.482320Z","shell.execute_reply":"2023-11-17T08:12:07.813689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_idx = \"m12\"\nnum_epochs = 5\nfor epoch in range(num_epochs):\n    # Train for one epoch:\n    train_one_epoch(model, optimizer, train_loader, device, epoch, print_freq=25)\n    # Note: The above function modifies the model in-place.\n    # Update the learning rate at the end of each epoch:\n    lr_scheduler.step()\n    # Evaluate on the validation set:\n    evaluate(model, val_loader, device=device)\n    # Compute the competition metric:\n    competition_metric = compute_metric()\n    print(f\"Competition metric: {competition_metric}\")\n    print(\"Saving model weights...\")\n    torch.save(model.state_dict(), f\"{model_idx}_{epoch}.pth\")\n    print(\"Model weights saved.\")\n    print(\"---\")\nprint(\"That's it!\")","metadata":{"execution":{"iopub.status.busy":"2023-11-17T08:12:26.049424Z","iopub.execute_input":"2023-11-17T08:12:26.050099Z","iopub.status.idle":"2023-11-17T10:22:14.566746Z","shell.execute_reply.started":"2023-11-17T08:12:26.050064Z","shell.execute_reply":"2023-11-17T10:22:14.565706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Examining a few validation set predictions","metadata":{}},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"m12_4.pth\"))\nmodel.eval();","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:22:51.675604Z","iopub.execute_input":"2023-11-17T10:22:51.676013Z","iopub.status.idle":"2023-11-17T10:22:51.833694Z","shell.execute_reply.started":"2023-11-17T10:22:51.675982Z","shell.execute_reply":"2023-11-17T10:22:51.832682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examining a couple of randomly selected images that don't have any annotations","metadata":{}},{"cell_type":"code","source":"image_names_with_annotations = annotations_df['image_name'].unique()\nval_without_annotations = val.loc[~ val['image_name'].isin(image_names_with_annotations)].reset_index(drop=True)\nval_without_annotations","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:06.993518Z","iopub.execute_input":"2023-11-17T10:23:06.994440Z","iopub.status.idle":"2023-11-17T10:23:07.013151Z","shell.execute_reply.started":"2023-11-17T10:23:06.994405Z","shell.execute_reply":"2023-11-17T10:23:07.012332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nidxs = np.random.randint(low=0, high=517, size=2)\nidxs","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:18.338418Z","iopub.execute_input":"2023-11-17T10:23:18.338806Z","iopub.status.idle":"2023-11-17T10:23:18.347386Z","shell.execute_reply.started":"2023-11-17T10:23:18.338776Z","shell.execute_reply":"2023-11-17T10:23:18.346555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### '2654a87be968_2018-10-27T20:30:00.jpg'","metadata":{}},{"cell_type":"code","source":"image_name = val_without_annotations['image_name'][idxs[0]]\nimage_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:24.934355Z","iopub.execute_input":"2023-11-17T10:23:24.934736Z","iopub.status.idle":"2023-11-17T10:23:24.941125Z","shell.execute_reply.started":"2023-11-17T10:23:24.934707Z","shell.execute_reply":"2023-11-17T10:23:24.940189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = os.path.join(data_prep_folder, \"images\", image_name)\nimage = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:33.042428Z","iopub.execute_input":"2023-11-17T10:23:33.043352Z","iopub.status.idle":"2023-11-17T10:23:33.051781Z","shell.execute_reply.started":"2023-11-17T10:23:33.043313Z","shell.execute_reply":"2023-11-17T10:23:33.050886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:33.428574Z","iopub.execute_input":"2023-11-17T10:23:33.429427Z","iopub.status.idle":"2023-11-17T10:23:33.441809Z","shell.execute_reply.started":"2023-11-17T10:23:33.429397Z","shell.execute_reply":"2023-11-17T10:23:33.440852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14.4, 4))\nplt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:37.523629Z","iopub.execute_input":"2023-11-17T10:23:37.524029Z","iopub.status.idle":"2023-11-17T10:23:38.030020Z","shell.execute_reply.started":"2023-11-17T10:23:37.524000Z","shell.execute_reply":"2023-11-17T10:23:38.029052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    x = eval_transforms(image)\n    x = x.to(device)\n    predictions = model([x])\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:42.043722Z","iopub.execute_input":"2023-11-17T10:23:42.044103Z","iopub.status.idle":"2023-11-17T10:23:42.110456Z","shell.execute_reply.started":"2023-11-17T10:23:42.044075Z","shell.execute_reply":"2023-11-17T10:23:42.109525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predictions[0]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:23:54.304621Z","iopub.execute_input":"2023-11-17T10:23:54.305043Z","iopub.status.idle":"2023-11-17T10:23:54.311789Z","shell.execute_reply.started":"2023-11-17T10:23:54.305013Z","shell.execute_reply":"2023-11-17T10:23:54.310700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_boxes = pred['boxes'].long()\n# pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n# output_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\n# plt.figure(figsize=(14.4, 4))\n# plt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:20.636275Z","iopub.execute_input":"2023-11-17T10:24:20.637162Z","iopub.status.idle":"2023-11-17T10:24:20.641311Z","shell.execute_reply.started":"2023-11-17T10:24:20.637125Z","shell.execute_reply":"2023-11-17T10:24:20.640397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 'ea0770830757_2018-10-21T20:30:00.jpg'","metadata":{}},{"cell_type":"code","source":"image_name = val_without_annotations['image_name'][idxs[1]]\nimage_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:25.769398Z","iopub.execute_input":"2023-11-17T10:24:25.770149Z","iopub.status.idle":"2023-11-17T10:24:25.776124Z","shell.execute_reply.started":"2023-11-17T10:24:25.770117Z","shell.execute_reply":"2023-11-17T10:24:25.775195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = os.path.join(data_prep_folder, \"images\", image_name)\nimage = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:26.981496Z","iopub.execute_input":"2023-11-17T10:24:26.982237Z","iopub.status.idle":"2023-11-17T10:24:26.990801Z","shell.execute_reply.started":"2023-11-17T10:24:26.982203Z","shell.execute_reply":"2023-11-17T10:24:26.990034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:30.191859Z","iopub.execute_input":"2023-11-17T10:24:30.192586Z","iopub.status.idle":"2023-11-17T10:24:30.204457Z","shell.execute_reply.started":"2023-11-17T10:24:30.192556Z","shell.execute_reply":"2023-11-17T10:24:30.203561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14.4, 4))\nplt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:30.487954Z","iopub.execute_input":"2023-11-17T10:24:30.488692Z","iopub.status.idle":"2023-11-17T10:24:30.949032Z","shell.execute_reply.started":"2023-11-17T10:24:30.488637Z","shell.execute_reply":"2023-11-17T10:24:30.948093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    x = eval_transforms(image)\n    x = x.to(device)\n    predictions = model([x])\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:35.348715Z","iopub.execute_input":"2023-11-17T10:24:35.349314Z","iopub.status.idle":"2023-11-17T10:24:35.417884Z","shell.execute_reply.started":"2023-11-17T10:24:35.349282Z","shell.execute_reply":"2023-11-17T10:24:35.416893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predictions[0]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:24:39.409945Z","iopub.execute_input":"2023-11-17T10:24:39.410325Z","iopub.status.idle":"2023-11-17T10:24:39.419380Z","shell.execute_reply.started":"2023-11-17T10:24:39.410293Z","shell.execute_reply":"2023-11-17T10:24:39.418460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thresh = 0.7\npred['boxes'] = pred['boxes'][pred['scores'] > thresh]\npred['labels'] = pred['labels'][pred['scores'] > thresh]\npred['scores'] = pred['scores'][pred['scores'] > thresh]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:26:22.698934Z","iopub.execute_input":"2023-11-17T10:26:22.699688Z","iopub.status.idle":"2023-11-17T10:26:22.707930Z","shell.execute_reply.started":"2023-11-17T10:26:22.699644Z","shell.execute_reply":"2023-11-17T10:26:22.706999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_boxes = pred['boxes'].long()\n# pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n# output_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\n# plt.figure(figsize=(14.4, 4))\n# plt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:26:39.985623Z","iopub.execute_input":"2023-11-17T10:26:39.986016Z","iopub.status.idle":"2023-11-17T10:26:39.990452Z","shell.execute_reply.started":"2023-11-17T10:26:39.985986Z","shell.execute_reply":"2023-11-17T10:26:39.989494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examining a couple of randomly selected images that have exactly 1 annotation","metadata":{}},{"cell_type":"code","source":"annotations_per_image = annotations_df.groupby('image_name').size()\nannotations_per_image","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:26:48.047279Z","iopub.execute_input":"2023-11-17T10:26:48.047677Z","iopub.status.idle":"2023-11-17T10:26:48.061893Z","shell.execute_reply.started":"2023-11-17T10:26:48.047633Z","shell.execute_reply":"2023-11-17T10:26:48.060995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"exactly_1_annotation = annotations_per_image.loc[annotations_per_image == 1]\nexactly_1_annotation","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:26:51.821267Z","iopub.execute_input":"2023-11-17T10:26:51.822176Z","iopub.status.idle":"2023-11-17T10:26:51.830042Z","shell.execute_reply.started":"2023-11-17T10:26:51.822140Z","shell.execute_reply":"2023-11-17T10:26:51.829021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_exactly_1_annotation = val.loc[val['image_name'].isin(exactly_1_annotation.index)].reset_index(drop=True)\nval_exactly_1_annotation","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:26:55.257913Z","iopub.execute_input":"2023-11-17T10:26:55.258942Z","iopub.status.idle":"2023-11-17T10:26:55.271130Z","shell.execute_reply.started":"2023-11-17T10:26:55.258908Z","shell.execute_reply":"2023-11-17T10:26:55.270143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nidxs = np.random.randint(low=0, high=1, size=2)\nidxs","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:06.731743Z","iopub.execute_input":"2023-11-17T10:27:06.732607Z","iopub.status.idle":"2023-11-17T10:27:06.739398Z","shell.execute_reply.started":"2023-11-17T10:27:06.732574Z","shell.execute_reply":"2023-11-17T10:27:06.738330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 'efbfc4526d58_2018-12-21T20:30:00.jpg'","metadata":{}},{"cell_type":"code","source":"image_name = val_exactly_1_annotation['image_name'][idxs[0]]\nimage_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:12.789733Z","iopub.execute_input":"2023-11-17T10:27:12.790975Z","iopub.status.idle":"2023-11-17T10:27:12.797437Z","shell.execute_reply.started":"2023-11-17T10:27:12.790934Z","shell.execute_reply":"2023-11-17T10:27:12.796483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = os.path.join(data_prep_folder, \"images\", image_name)\nimage = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:13.091187Z","iopub.execute_input":"2023-11-17T10:27:13.091550Z","iopub.status.idle":"2023-11-17T10:27:13.101570Z","shell.execute_reply.started":"2023-11-17T10:27:13.091523Z","shell.execute_reply":"2023-11-17T10:27:13.100707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:15.818741Z","iopub.execute_input":"2023-11-17T10:27:15.819110Z","iopub.status.idle":"2023-11-17T10:27:15.834851Z","shell.execute_reply.started":"2023-11-17T10:27:15.819081Z","shell.execute_reply":"2023-11-17T10:27:15.833852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window['x'] = (window['x0'] + window['x1']) / 2\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:17.305751Z","iopub.execute_input":"2023-11-17T10:27:17.306527Z","iopub.status.idle":"2023-11-17T10:27:17.320091Z","shell.execute_reply.started":"2023-11-17T10:27:17.306496Z","shell.execute_reply":"2023-11-17T10:27:17.319110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:20.349616Z","iopub.execute_input":"2023-11-17T10:27:20.350516Z","iopub.status.idle":"2023-11-17T10:27:20.881873Z","shell.execute_reply.started":"2023-11-17T10:27:20.350485Z","shell.execute_reply":"2023-11-17T10:27:20.880886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    x = eval_transforms(image)\n    x = x.to(device)\n    predictions = model([x])\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:25.268360Z","iopub.execute_input":"2023-11-17T10:27:25.269276Z","iopub.status.idle":"2023-11-17T10:27:25.339371Z","shell.execute_reply.started":"2023-11-17T10:27:25.269232Z","shell.execute_reply":"2023-11-17T10:27:25.338130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predictions[0]\npred['boxes'] = pred['boxes'][pred['scores'] > thresh]\npred['labels'] = pred['labels'][pred['scores'] > thresh]\npred['scores'] = pred['scores'][pred['scores'] > thresh]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:39.762417Z","iopub.execute_input":"2023-11-17T10:27:39.762808Z","iopub.status.idle":"2023-11-17T10:27:39.773611Z","shell.execute_reply.started":"2023-11-17T10:27:39.762776Z","shell.execute_reply":"2023-11-17T10:27:39.772763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_boxes = pred['boxes'].long()\npred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\noutput_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\nplt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:44.130547Z","iopub.execute_input":"2023-11-17T10:27:44.131259Z","iopub.status.idle":"2023-11-17T10:27:44.604798Z","shell.execute_reply.started":"2023-11-17T10:27:44.131224Z","shell.execute_reply":"2023-11-17T10:27:44.603813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NA","metadata":{}},{"cell_type":"code","source":"# image_name = val_exactly_1_annotation['image_name'][idxs[1]]\n# image_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:56.539286Z","iopub.execute_input":"2023-11-17T10:27:56.539683Z","iopub.status.idle":"2023-11-17T10:27:56.543953Z","shell.execute_reply.started":"2023-11-17T10:27:56.539641Z","shell.execute_reply":"2023-11-17T10:27:56.542931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_path = os.path.join(data_prep_folder, \"images\", image_name)\n# image = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:56.825863Z","iopub.execute_input":"2023-11-17T10:27:56.826226Z","iopub.status.idle":"2023-11-17T10:27:56.830674Z","shell.execute_reply.started":"2023-11-17T10:27:56.826198Z","shell.execute_reply":"2023-11-17T10:27:56.829633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:57.244455Z","iopub.execute_input":"2023-11-17T10:27:57.244861Z","iopub.status.idle":"2023-11-17T10:27:57.249167Z","shell.execute_reply.started":"2023-11-17T10:27:57.244828Z","shell.execute_reply":"2023-11-17T10:27:57.248202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window['x'] = (window['x0'] + window['x1']) / 2\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:57.866241Z","iopub.execute_input":"2023-11-17T10:27:57.866617Z","iopub.status.idle":"2023-11-17T10:27:57.870769Z","shell.execute_reply.started":"2023-11-17T10:27:57.866588Z","shell.execute_reply":"2023-11-17T10:27:57.869815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:58.288482Z","iopub.execute_input":"2023-11-17T10:27:58.289254Z","iopub.status.idle":"2023-11-17T10:27:58.293615Z","shell.execute_reply.started":"2023-11-17T10:27:58.289213Z","shell.execute_reply":"2023-11-17T10:27:58.292620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with torch.no_grad():\n#     x = eval_transforms(image)\n#     x = x.to(device)\n#     predictions = model([x])\n# predictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:59.425645Z","iopub.execute_input":"2023-11-17T10:27:59.426044Z","iopub.status.idle":"2023-11-17T10:27:59.430257Z","shell.execute_reply.started":"2023-11-17T10:27:59.426015Z","shell.execute_reply":"2023-11-17T10:27:59.429258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred = predictions[0]\n# pred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:27:59.953586Z","iopub.execute_input":"2023-11-17T10:27:59.954726Z","iopub.status.idle":"2023-11-17T10:27:59.958347Z","shell.execute_reply.started":"2023-11-17T10:27:59.954689Z","shell.execute_reply":"2023-11-17T10:27:59.957527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_boxes = pred['boxes'].long()\n# pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n# output_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\n# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:00.392948Z","iopub.execute_input":"2023-11-17T10:28:00.393323Z","iopub.status.idle":"2023-11-17T10:28:00.397810Z","shell.execute_reply.started":"2023-11-17T10:28:00.393295Z","shell.execute_reply":"2023-11-17T10:28:00.396797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examining a couple of randomly selected images that have exactly 2 annotations","metadata":{}},{"cell_type":"code","source":"exactly_2_annotations = annotations_per_image.loc[annotations_per_image == 2]\nexactly_2_annotations","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:43.200837Z","iopub.execute_input":"2023-11-17T10:28:43.201286Z","iopub.status.idle":"2023-11-17T10:28:43.211367Z","shell.execute_reply.started":"2023-11-17T10:28:43.201249Z","shell.execute_reply":"2023-11-17T10:28:43.210458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_exactly_2_annotations = val.loc[val['image_name'].isin(exactly_2_annotations.index)].reset_index(drop=True)\nval_exactly_2_annotations","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:43.737547Z","iopub.execute_input":"2023-11-17T10:28:43.737954Z","iopub.status.idle":"2023-11-17T10:28:43.755633Z","shell.execute_reply.started":"2023-11-17T10:28:43.737924Z","shell.execute_reply":"2023-11-17T10:28:43.754628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nidxs = np.random.randint(low=0, high=954, size=2)\nidxs","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:48.577780Z","iopub.execute_input":"2023-11-17T10:28:48.578517Z","iopub.status.idle":"2023-11-17T10:28:48.585114Z","shell.execute_reply.started":"2023-11-17T10:28:48.578486Z","shell.execute_reply":"2023-11-17T10:28:48.584040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### '137b99e936ab_2018-07-25T20:30:00.jpg'","metadata":{}},{"cell_type":"code","source":"image_name = val_exactly_2_annotations['image_name'][idxs[0]]\nimage_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:51.824042Z","iopub.execute_input":"2023-11-17T10:28:51.824934Z","iopub.status.idle":"2023-11-17T10:28:51.831227Z","shell.execute_reply.started":"2023-11-17T10:28:51.824903Z","shell.execute_reply":"2023-11-17T10:28:51.830245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = os.path.join(data_prep_folder, \"images\", image_name)\nimage = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:56.646081Z","iopub.execute_input":"2023-11-17T10:28:56.646821Z","iopub.status.idle":"2023-11-17T10:28:56.657711Z","shell.execute_reply.started":"2023-11-17T10:28:56.646751Z","shell.execute_reply":"2023-11-17T10:28:56.656899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:56.954045Z","iopub.execute_input":"2023-11-17T10:28:56.954743Z","iopub.status.idle":"2023-11-17T10:28:56.969696Z","shell.execute_reply.started":"2023-11-17T10:28:56.954707Z","shell.execute_reply":"2023-11-17T10:28:56.968727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window['x'] = (window['x0'] + window['x1']) / 2\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:28:59.976381Z","iopub.execute_input":"2023-11-17T10:28:59.977481Z","iopub.status.idle":"2023-11-17T10:28:59.992203Z","shell.execute_reply.started":"2023-11-17T10:28:59.977446Z","shell.execute_reply":"2023-11-17T10:28:59.991249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:00.057445Z","iopub.execute_input":"2023-11-17T10:29:00.057748Z","iopub.status.idle":"2023-11-17T10:29:00.596243Z","shell.execute_reply.started":"2023-11-17T10:29:00.057723Z","shell.execute_reply":"2023-11-17T10:29:00.595345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    x = eval_transforms(image)\n    x = x.to(device)\n    predictions = model([x])\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:04.799485Z","iopub.execute_input":"2023-11-17T10:29:04.800252Z","iopub.status.idle":"2023-11-17T10:29:04.871412Z","shell.execute_reply.started":"2023-11-17T10:29:04.800218Z","shell.execute_reply":"2023-11-17T10:29:04.870502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predictions[0]\npred['boxes'] = pred['boxes'][pred['scores'] > thresh]\npred['labels'] = pred['labels'][pred['scores'] > thresh]\npred['scores'] = pred['scores'][pred['scores'] > thresh]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:28.836388Z","iopub.execute_input":"2023-11-17T10:29:28.836821Z","iopub.status.idle":"2023-11-17T10:29:28.849701Z","shell.execute_reply.started":"2023-11-17T10:29:28.836788Z","shell.execute_reply":"2023-11-17T10:29:28.848399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_boxes = pred['boxes'].long()\npred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\noutput_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\nplt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:32.202178Z","iopub.execute_input":"2023-11-17T10:29:32.202567Z","iopub.status.idle":"2023-11-17T10:29:32.682165Z","shell.execute_reply.started":"2023-11-17T10:29:32.202535Z","shell.execute_reply":"2023-11-17T10:29:32.681237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### '72bbd1ac3edf_2017-10-08T20:30:00.jpg'","metadata":{}},{"cell_type":"code","source":"image_name = val_exactly_2_annotations['image_name'][idxs[1]]\nimage_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:39.923004Z","iopub.execute_input":"2023-11-17T10:29:39.923742Z","iopub.status.idle":"2023-11-17T10:29:39.929691Z","shell.execute_reply.started":"2023-11-17T10:29:39.923705Z","shell.execute_reply":"2023-11-17T10:29:39.928738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image_path = os.path.join(data_prep_folder, \"images\", image_name)\nimage = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:43.494770Z","iopub.execute_input":"2023-11-17T10:29:43.495545Z","iopub.status.idle":"2023-11-17T10:29:43.504904Z","shell.execute_reply.started":"2023-11-17T10:29:43.495511Z","shell.execute_reply":"2023-11-17T10:29:43.504064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:43.613254Z","iopub.execute_input":"2023-11-17T10:29:43.613557Z","iopub.status.idle":"2023-11-17T10:29:43.629472Z","shell.execute_reply.started":"2023-11-17T10:29:43.613532Z","shell.execute_reply":"2023-11-17T10:29:43.628474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"window['x'] = (window['x0'] + window['x1']) / 2\nwindow","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:44.020286Z","iopub.execute_input":"2023-11-17T10:29:44.020644Z","iopub.status.idle":"2023-11-17T10:29:44.036162Z","shell.execute_reply.started":"2023-11-17T10:29:44.020617Z","shell.execute_reply":"2023-11-17T10:29:44.035046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:46.736496Z","iopub.execute_input":"2023-11-17T10:29:46.737139Z","iopub.status.idle":"2023-11-17T10:29:47.187577Z","shell.execute_reply.started":"2023-11-17T10:29:46.737109Z","shell.execute_reply":"2023-11-17T10:29:47.186614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    x = eval_transforms(image)\n    x = x.to(device)\n    predictions = model([x])\npredictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:50.838204Z","iopub.execute_input":"2023-11-17T10:29:50.838559Z","iopub.status.idle":"2023-11-17T10:29:50.907907Z","shell.execute_reply.started":"2023-11-17T10:29:50.838534Z","shell.execute_reply":"2023-11-17T10:29:50.906940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = predictions[0]\npred['boxes'] = pred['boxes'][pred['scores'] > thresh]\npred['labels'] = pred['labels'][pred['scores'] > thresh]\npred['scores'] = pred['scores'][pred['scores'] > thresh]\npred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:57.341783Z","iopub.execute_input":"2023-11-17T10:29:57.342521Z","iopub.status.idle":"2023-11-17T10:29:57.352661Z","shell.execute_reply.started":"2023-11-17T10:29:57.342491Z","shell.execute_reply":"2023-11-17T10:29:57.351802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_boxes = pred['boxes'].long()\npred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\noutput_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\nplt.figure(figsize=(14.4, 4))\nplt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\nplt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:29:57.848223Z","iopub.execute_input":"2023-11-17T10:29:57.848612Z","iopub.status.idle":"2023-11-17T10:29:58.366062Z","shell.execute_reply.started":"2023-11-17T10:29:57.848581Z","shell.execute_reply":"2023-11-17T10:29:58.365111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Examining a couple of randomly selected images that have exactly 3 annotations","metadata":{}},{"cell_type":"code","source":"exactly_3_annotations = annotations_per_image.loc[annotations_per_image == 3]\nexactly_3_annotations","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:05.325137Z","iopub.execute_input":"2023-11-17T10:30:05.325802Z","iopub.status.idle":"2023-11-17T10:30:05.333450Z","shell.execute_reply.started":"2023-11-17T10:30:05.325762Z","shell.execute_reply":"2023-11-17T10:30:05.332506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_exactly_3_annotations = val.loc[val['image_name'].isin(exactly_3_annotations.index)].reset_index(drop=True)\nval_exactly_3_annotations","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:07.156107Z","iopub.execute_input":"2023-11-17T10:30:07.156890Z","iopub.status.idle":"2023-11-17T10:30:07.166865Z","shell.execute_reply.started":"2023-11-17T10:30:07.156858Z","shell.execute_reply":"2023-11-17T10:30:07.165725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# np.random.seed(42)\n# idxs = np.random.randint(low=0, high=4, size=2)\n# idxs","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:10.317591Z","iopub.execute_input":"2023-11-17T10:30:10.318005Z","iopub.status.idle":"2023-11-17T10:30:10.322292Z","shell.execute_reply.started":"2023-11-17T10:30:10.317975Z","shell.execute_reply":"2023-11-17T10:30:10.321205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NA","metadata":{}},{"cell_type":"code","source":"# image_name = val_exactly_3_annotations['image_name'][idxs[0]]\n# image_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:35.761858Z","iopub.execute_input":"2023-11-17T10:30:35.762763Z","iopub.status.idle":"2023-11-17T10:30:35.766734Z","shell.execute_reply.started":"2023-11-17T10:30:35.762730Z","shell.execute_reply":"2023-11-17T10:30:35.765632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_path = os.path.join(data_prep_folder, \"images\", image_name)\n# image = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:36.102686Z","iopub.execute_input":"2023-11-17T10:30:36.103569Z","iopub.status.idle":"2023-11-17T10:30:36.107719Z","shell.execute_reply.started":"2023-11-17T10:30:36.103536Z","shell.execute_reply":"2023-11-17T10:30:36.106691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:39.335675Z","iopub.execute_input":"2023-11-17T10:30:39.336556Z","iopub.status.idle":"2023-11-17T10:30:39.340545Z","shell.execute_reply.started":"2023-11-17T10:30:39.336521Z","shell.execute_reply":"2023-11-17T10:30:39.339557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window['x'] = (window['x0'] + window['x1']) / 2\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:39.860184Z","iopub.execute_input":"2023-11-17T10:30:39.861097Z","iopub.status.idle":"2023-11-17T10:30:39.865070Z","shell.execute_reply.started":"2023-11-17T10:30:39.861063Z","shell.execute_reply":"2023-11-17T10:30:39.864091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:41.768490Z","iopub.execute_input":"2023-11-17T10:30:41.769377Z","iopub.status.idle":"2023-11-17T10:30:41.773548Z","shell.execute_reply.started":"2023-11-17T10:30:41.769345Z","shell.execute_reply":"2023-11-17T10:30:41.772542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with torch.no_grad():\n#     x = eval_transforms(image)\n#     x = x.to(device)\n#     predictions = model([x])\n# predictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:41.885353Z","iopub.execute_input":"2023-11-17T10:30:41.886273Z","iopub.status.idle":"2023-11-17T10:30:41.890398Z","shell.execute_reply.started":"2023-11-17T10:30:41.886237Z","shell.execute_reply":"2023-11-17T10:30:41.889365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred = predictions[0]\n# pred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:44.386397Z","iopub.execute_input":"2023-11-17T10:30:44.387051Z","iopub.status.idle":"2023-11-17T10:30:44.391088Z","shell.execute_reply.started":"2023-11-17T10:30:44.387019Z","shell.execute_reply":"2023-11-17T10:30:44.390077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_boxes = pred['boxes'].long()\n# pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n# output_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\n# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:44.535981Z","iopub.execute_input":"2023-11-17T10:30:44.536743Z","iopub.status.idle":"2023-11-17T10:30:44.540836Z","shell.execute_reply.started":"2023-11-17T10:30:44.536710Z","shell.execute_reply":"2023-11-17T10:30:44.539848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### NA","metadata":{}},{"cell_type":"code","source":"# image_name = val_exactly_3_annotations['image_name'][idxs[1]]\n# image_name","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:54.433083Z","iopub.execute_input":"2023-11-17T10:30:54.433460Z","iopub.status.idle":"2023-11-17T10:30:54.437553Z","shell.execute_reply.started":"2023-11-17T10:30:54.433431Z","shell.execute_reply":"2023-11-17T10:30:54.436543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image_path = os.path.join(data_prep_folder, \"images\", image_name)\n# image = read_image(image_path)","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:54.776125Z","iopub.execute_input":"2023-11-17T10:30:54.777164Z","iopub.status.idle":"2023-11-17T10:30:54.781355Z","shell.execute_reply.started":"2023-11-17T10:30:54.777119Z","shell.execute_reply":"2023-11-17T10:30:54.780314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window = annotations_df.loc[annotations_df['image_name'] == image_name].reset_index(drop=True)\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:55.138624Z","iopub.execute_input":"2023-11-17T10:30:55.139363Z","iopub.status.idle":"2023-11-17T10:30:55.143193Z","shell.execute_reply.started":"2023-11-17T10:30:55.139332Z","shell.execute_reply":"2023-11-17T10:30:55.142149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# window['x'] = (window['x0'] + window['x1']) / 2\n# window","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:55.392973Z","iopub.execute_input":"2023-11-17T10:30:55.393355Z","iopub.status.idle":"2023-11-17T10:30:55.397738Z","shell.execute_reply.started":"2023-11-17T10:30:55.393324Z","shell.execute_reply":"2023-11-17T10:30:55.396615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:55.982317Z","iopub.execute_input":"2023-11-17T10:30:55.983216Z","iopub.status.idle":"2023-11-17T10:30:55.987143Z","shell.execute_reply.started":"2023-11-17T10:30:55.983182Z","shell.execute_reply":"2023-11-17T10:30:55.986017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with torch.no_grad():\n#     x = eval_transforms(image)\n#     x = x.to(device)\n#     predictions = model([x])\n# predictions","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:56.232846Z","iopub.execute_input":"2023-11-17T10:30:56.233490Z","iopub.status.idle":"2023-11-17T10:30:56.237624Z","shell.execute_reply.started":"2023-11-17T10:30:56.233459Z","shell.execute_reply":"2023-11-17T10:30:56.236535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred = predictions[0]\n# pred","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:56.964917Z","iopub.execute_input":"2023-11-17T10:30:56.965296Z","iopub.status.idle":"2023-11-17T10:30:56.969327Z","shell.execute_reply.started":"2023-11-17T10:30:56.965267Z","shell.execute_reply":"2023-11-17T10:30:56.968432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pred_boxes = pred['boxes'].long()\n# pred_labels = [reverse_label_mapping[l.item()] for l in pred['labels']]\n# output_image = draw_bounding_boxes(image, pred_boxes, pred_labels, colors=\"cyan\", width=4)\n# plt.figure(figsize=(14.4, 4))\n# plt.vlines(x=window['x'], ymin=0, ymax=400, colors=\"yellow\", linewidth=4)\n# plt.imshow(output_image.permute(1, 2, 0));","metadata":{"execution":{"iopub.status.busy":"2023-11-17T10:30:57.152998Z","iopub.execute_input":"2023-11-17T10:30:57.154030Z","iopub.status.idle":"2023-11-17T10:30:57.158589Z","shell.execute_reply.started":"2023-11-17T10:30:57.153983Z","shell.execute_reply":"2023-11-17T10:30:57.157715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---\n\n**Reference:** https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}