{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13719397,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MABe Challenge - Social Action Recognition in Mice: Validated baseline without machine learning\n\nThis notebook has two purposes:\n1. It shows how to produce predictions without machine learning.\n1. It provides the code for validating out-of-fold predictions with the competition's `score()` function.\n\nReferences\n- Competition: [MABe Challenge - Social Action Recognition in Mice](https://www.kaggle.com/competitions/MABe-mouse-behavior-detection)\n- EDA: [MABe EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-eda-which-makes-sense)\n- ML model: [MABe Nearest Neighbors: The Original ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-nearest-neighbors-the-original)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:09.245461Z","iopub.execute_input":"2025-09-27T12:12:09.245830Z","iopub.status.idle":"2025-09-27T12:12:09.773532Z","shell.execute_reply.started":"2025-09-27T12:12:09.245803Z","shell.execute_reply":"2025-09-27T12:12:09.771622Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"F Beta customized for the data format of the MABe challenge.\"\"\"\n\nimport json\n\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\n\n\nclass HostVisibleError(Exception):\n    pass\n\n\ndef single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n    label_frames: defaultdict[str, set[int]] = defaultdict(set) # key is video/agent/target/action from solution\n    prediction_frames: defaultdict[str, set[int]] = defaultdict(set) # key is video/agent/target/action from submission\n\n    for row in lab_solution.to_dicts():\n        label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n    for video in lab_solution['video_id'].unique():\n        active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first()  # ty: ignore\n        active_labels: set[str] = set(json.loads(active_labels)) # set of agent,target,action from solution\n        predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set) # key is agent,target from submission\n\n        for row in lab_submission.filter(pl.col('video_id') == video).to_dicts(): # every submission row is converted to a dict\n            # Since the labels are sparse, we can't evaluate prediction keys not in the active labels.\n            if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n                continue # these submission rows are ignored\n           \n            new_frames = set(range(row['start_frame'], row['stop_frame']))\n            # Ignore truly redundant predictions.\n            new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n            prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n            if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n                # A single agent can have multiple targets per frame (ex: evading all other mice) but only one action per target per frame.\n                raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n            prediction_frames[row['prediction_key']].update(new_frames)\n            predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n    tps = defaultdict(int) # key is action\n    fns = defaultdict(int) # key is action\n    fps = defaultdict(int) # key is action\n    for key, pred_frames in prediction_frames.items():\n        action = key.split('_')[-1]\n        matched_label_frames = label_frames[key]\n        tps[action] += len(pred_frames.intersection(matched_label_frames))\n        fns[action] += len(matched_label_frames.difference(pred_frames))\n        fps[action] += len(pred_frames.difference(matched_label_frames))\n\n    distinct_actions = set()\n    for key, frames in label_frames.items():\n        action = key.split('_')[-1]\n        distinct_actions.add(action)\n        if key not in prediction_frames:\n            fns[action] += len(frames)\n\n    action_f1s = []\n    for action in distinct_actions:\n        # print(f\"{tps[action]:8} {fns[action]:8} {fps[action]:8}\")\n        if tps[action] + fns[action] + fps[action] == 0:\n            action_f1s.append(0)\n        else:\n            action_f1s.append((1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action]))\n    return sum(action_f1s) / len(action_f1s)\n\n\ndef mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n    \"\"\"\n    Doctests:\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    1.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 0, 'stop_frame': 10}, # Wrong action\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    0.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n    ... ])\n    >>> \"%.12f\" % mouse_fbeta(solution, submission)\n    '0.500000000000'\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n    ... ])\n    >>> \"%.12f\" % mouse_fbeta(solution, submission)\n    '0.250000000000'\n\n    >>> # Overlapping solution events, one prediction matching both.\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 10, 'stop_frame': 20, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 20},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    1.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 30, 'stop_frame': 40, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 40},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    0.6666666666666666\n    \"\"\"\n    if len(solution) == 0 or len(submission) == 0:\n        raise ValueError('Missing solution or submission data')\n\n    expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n    for col in expected_cols:\n        if col not in solution.columns:\n            raise ValueError(f'Solution is missing column {col}')\n        if col not in submission.columns:\n            raise ValueError(f'Submission is missing column {col}')\n\n    solution: pl.DataFrame = pl.DataFrame(solution)\n    submission: pl.DataFrame = pl.DataFrame(submission)\n    assert (solution['start_frame'] <= solution['stop_frame']).all()\n    assert (submission['start_frame'] <= submission['stop_frame']).all()\n    solution_videos = set(solution['video_id'].unique())\n    # Need to align based on video IDs as we can't rely on the row IDs for handling public/private splits.\n    submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n    solution = solution.with_columns(\n        pl.concat_str(\n            [\n                pl.col('video_id').cast(pl.Utf8),\n                pl.col('agent_id').cast(pl.Utf8),\n                pl.col('target_id').cast(pl.Utf8),\n                pl.col('action'),\n            ],\n            separator='_',\n        ).alias('label_key'),\n    )\n    submission = submission.with_columns(\n        pl.concat_str(\n            [\n                pl.col('video_id').cast(pl.Utf8),\n                pl.col('agent_id').cast(pl.Utf8),\n                pl.col('target_id').cast(pl.Utf8),\n                pl.col('action'),\n            ],\n            separator='_',\n        ).alias('prediction_key'),\n    )\n\n    lab_scores = []\n    for lab in solution['lab_id'].unique():\n        lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n        lab_videos = set(lab_solution['video_id'].unique())\n        lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n        # print(len(lab_solution), len(lab_videos), len(lab_submission), single_lab_f1(lab_solution, lab_submission, beta=beta))\n        lab_scores.append(single_lab_f1(lab_solution, lab_submission, beta=beta))\n\n    return sum(lab_scores) / len(lab_scores)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n    \"\"\"\n    F1 score for the MABe Challenge\n    \"\"\"\n    solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n    submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n    return mouse_fbeta(solution, submission, beta=beta)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:09.776141Z","iopub.execute_input":"2025-09-27T12:12:09.776675Z","iopub.status.idle":"2025-09-27T12:12:10.027713Z","shell.execute_reply.started":"2025-09-27T12:12:09.776646Z","shell.execute_reply":"2025-09-27T12:12:10.025845Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Validation\n\nWe first create the solution dataframe, which contains all true labels in the correct format for the `score()` function.","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/train.csv')\ntrain_subset = train.query(\"~ lab_id.str.startswith('MABe22_')\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:10.028696Z","iopub.execute_input":"2025-09-27T12:12:10.029022Z","iopub.status.idle":"2025-09-27T12:12:10.215237Z","shell.execute_reply.started":"2025-09-27T12:12:10.028996Z","shell.execute_reply":"2025-09-27T12:12:10.213275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_solution_df(dataset):\n    \"\"\"Create the solution dataframe for validating out-of-fold predictions.\n\n    From https://www.kaggle.com/code/ambrosm/mabe-validated-baseline-without-machine-learning/\n    \n    Parameters:\n    dataset: (a subset of) the train dataframe\n    \n    Return values:\n    solution: solution dataframe in the correct format for the score() function\n    \"\"\"\n    solution = []\n    for train_idx, row in tqdm(dataset.iterrows(), total=len(dataset)):\n    \n        # Load annotation file\n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'): continue\n        video_id = row['video_id']\n        path = f\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/{lab_id}/{video_id}.parquet\"\n        try:\n            annot = pd.read_parquet(path.replace('train_tracking', 'train_annotation'))\n        except FileNotFoundError:\n            print(f\"No annotations for {path}\")\n            continue\n    \n        # Add all annotations to the solution\n        self_annotation = annot.target_id == annot.agent_id\n        annot['lab_id'] = lab_id\n        annot['video_id'] = video_id\n        annot['behaviors_labeled'] = row['behaviors_labeled']\n        annot['agent_id'] = annot['agent_id'].apply(lambda s: f\"mouse{s}\")\n        annot['target_id'] = np.where(self_annotation, 'self', annot['target_id'].apply(lambda s: f\"mouse{s}\"))\n        solution.append(annot)\n    \n    solution = pd.concat(solution)\n    return solution\n\nsolution = create_solution_df(train_subset)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:10.218695Z","iopub.execute_input":"2025-09-27T12:12:10.219095Z","iopub.status.idle":"2025-09-27T12:12:17.519302Z","shell.execute_reply.started":"2025-09-27T12:12:10.219059Z","shell.execute_reply":"2025-09-27T12:12:17.517640Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Now we'll predict something. In binary classification without machine learning, you can either predict all true or all false:\n- If you predict all false, the f1 score will be 0 because the recall is 0.\n- If you predict all true, the recall will be 1 and the precision > 0 so that the f1 score (harmonic mean of recall and precision) will be positive.\n\nPredicting all true is obviously the better choice.\n\nHowever, this competition is not binary classification. For every video_id/agent/target combination, we may predict *at most one* of several actions. This is multi-class classification (every action is a class, and 'no-action' is an additional class). To get positive recalls and f1 scores for all actions, we distribute our predictions to all of them (but we never predict 'no-action'). Note that the set of possible actions depends on the video_id/agent/target combination.\n","metadata":{}},{"cell_type":"code","source":"def predict_without_ml(dataset, traintest):\n    \"\"\"Predict actions without machine learning\n    \n    Parameters:\n    dataset: (a subset of) the train or test dataframe\n    traintest: 'train' or 'test'\n    \n    Return values:\n    submission: submission dataframe\n    \"\"\"\n    submission = []\n    for train_idx, row in tqdm(dataset.iterrows(), total=len(dataset)):\n    \n        # Load video\n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'): continue\n        video_id = row['video_id']\n        path = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n    \n        # Determine the behaviors of this video\n        if type(row.behaviors_labeled) != str:\n            continue\n        vid_behaviors = json.loads(row['behaviors_labeled'])\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(',') for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n    \n        # Determine start_frame and stop_frame\n        start_frame = vid.video_frame.min()\n        stop_frame = vid.video_frame.max() + 1\n    \n        # Predict all possible actions as often as possible\n        for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n            batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n            for i, (_, action_row) in enumerate(actions.iterrows()):\n                batch_start = start_frame + i * batch_length\n                batch_stop = min(batch_start + batch_length, stop_frame)\n                submission.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n    \n    # Convert to dataframe\n    submission = pd.DataFrame(submission, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n\n    return submission\n\nsubmission = predict_without_ml(train_subset, 'train')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:17.521061Z","iopub.execute_input":"2025-09-27T12:12:17.521602Z","iopub.status.idle":"2025-09-27T12:12:44.424938Z","shell.execute_reply.started":"2025-09-27T12:12:17.521563Z","shell.execute_reply":"2025-09-27T12:12:44.423772Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"What score would we get on the training dataset?","metadata":{}},{"cell_type":"code","source":"%%time\nscore(solution, submission, '')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:44.426238Z","iopub.execute_input":"2025-09-27T12:12:44.426827Z","iopub.status.idle":"2025-09-27T12:12:57.106895Z","shell.execute_reply.started":"2025-09-27T12:12:44.426790Z","shell.execute_reply":"2025-09-27T12:12:57.105769Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/test.csv')\nsubmission = predict_without_ml(test, 'test')\nsubmission.index.name = 'row_id'\nsubmission.to_csv('submission.csv')\n!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T12:12:57.107833Z","iopub.execute_input":"2025-09-27T12:12:57.108189Z","iopub.status.idle":"2025-09-27T12:12:57.370822Z","shell.execute_reply.started":"2025-09-27T12:12:57.108156Z","shell.execute_reply":"2025-09-27T12:12:57.368862Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}