{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13719397,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ==================\n# import libraries\n# ==================\nimport os, gc, glob, pickle, warnings\nimport random, math, time\nimport joblib, pickle, itertools\nfrom pathlib import Path\nfrom collections import defaultdict\n\nimport numpy as np\nimport scipy as sp\nimport polars as pl\nimport pandas as pd\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:22:56.674147Z","iopub.execute_input":"2025-09-20T08:22:56.674415Z","iopub.status.idle":"2025-09-20T08:23:03.012668Z","shell.execute_reply.started":"2025-09-20T08:22:56.674390Z","shell.execute_reply":"2025-09-20T08:23:03.010940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===============\n# utils\n# ===============\ndef sep(word, num=80):\n    print(\"=\"*num); print(word); print(\"=\"*80)\n\ndef show_df(df, num=3, showtail=False):\n    print(df.shape)\n    display(df.head(num))\n    if showtail:\n        display(df.tail(num))\n\ndef glob_walk(root: Path, glob_str: str) -> list:\n    path = Path(root)\n    walker = sorted(list(path.glob(glob_str)))\n    return walker\n\ndef seed_everything(seed, GPU=False):\n    random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    np.random.seed(seed)\n    if GPU:\n        torch.manual_seed(seed)\n        torch.cuda.manual_seed(seed)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\nseed_everything(0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:07.828570Z","iopub.execute_input":"2025-09-20T08:23:07.829466Z","iopub.status.idle":"2025-09-20T08:23:07.839661Z","shell.execute_reply.started":"2025-09-20T08:23:07.829395Z","shell.execute_reply":"2025-09-20T08:23:07.837875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PATHS:\n    base_dir = Path(\"/kaggle/input/MABe-mouse-behavior-detection\")\n    train_tracking_dir = Path(\"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\")\n    train_annotation_dir = Path(\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\")\n\nprint(glob_walk(PATHS.base_dir, \"*\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:10.326604Z","iopub.execute_input":"2025-09-20T08:23:10.327077Z","iopub.status.idle":"2025-09-20T08:23:10.337181Z","shell.execute_reply.started":"2025-09-20T08:23:10.327051Z","shell.execute_reply":"2025-09-20T08:23:10.335218Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Check tain.csv, test.csv, sample_submission.csv\n---\n- [train/test].csv Metadata about the mice and recording setups.\n- sample_submission.csv A submission file in the correct format.","metadata":{}},{"cell_type":"code","source":"# ======================\n# Raed data(submission)\n# ======================\nsub_df = pl.read_csv(PATHS.base_dir/\"sample_submission.csv\")\nsep(\"sample_submission.csv\"); show_df(sub_df, 3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:14.693289Z","iopub.execute_input":"2025-09-20T08:23:14.693671Z","iopub.status.idle":"2025-09-20T08:23:14.831247Z","shell.execute_reply.started":"2025-09-20T08:23:14.693648Z","shell.execute_reply":"2025-09-20T08:23:14.829968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# Raed data(test)\n# ======================\ntest_df = pl.read_csv(PATHS.base_dir/\"test.csv\")\nsep(\"test.csv\"); show_df(test_df, 3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:19.013414Z","iopub.execute_input":"2025-09-20T08:23:19.015008Z","iopub.status.idle":"2025-09-20T08:23:19.033183Z","shell.execute_reply.started":"2025-09-20T08:23:19.014941Z","shell.execute_reply":"2025-09-20T08:23:19.031861Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======================\n# Raed data\n# ======================\ntrain_df = pl.read_csv(PATHS.base_dir/\"train.csv\")\nsep(\"train.csv\"); show_df(train_df, 3), display(train_df.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:19.920536Z","iopub.execute_input":"2025-09-20T08:23:19.920862Z","iopub.status.idle":"2025-09-20T08:23:20.129801Z","shell.execute_reply.started":"2025-09-20T08:23:19.920837Z","shell.execute_reply":"2025-09-20T08:23:20.128894Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================\n# check columns\n# =================\nsub_cols   = list(sub_df.columns)\ntest_cols  = list(test_df.columns)\ntrain_cols = list(train_df.columns)\n\nsep(\"train_cols\")\nprint(train_cols)\n\nsep(\"cols in train_cols which is not in test_cols\")\nprint([col for col in train_cols if col not in test_cols])\n\nsep(\"cols in sub_cols which is not in train_cols\")\nprint([col for col in sub_cols if col not in train_cols])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:22.474209Z","iopub.execute_input":"2025-09-20T08:23:22.474544Z","iopub.status.idle":"2025-09-20T08:23:22.482234Z","shell.execute_reply.started":"2025-09-20T08:23:22.474520Z","shell.execute_reply":"2025-09-20T08:23:22.480853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =================\n# lab_id\n# =================\nunique_labid = list(train_df[\"lab_id\"].unique())\n\nsep(\"lab_id info\")\nprint(f\"n_unique (lab_id): {len(unique_labid)}\")\nprint(unique_labid)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:25.805633Z","iopub.execute_input":"2025-09-20T08:23:25.806088Z","iopub.status.idle":"2025-09-20T08:23:25.818346Z","shell.execute_reply.started":"2025-09-20T08:23:25.806051Z","shell.execute_reply":"2025-09-20T08:23:25.817340Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# =====================\n# Count per lab_id\n# =====================\nlabid_counts = (\n    train_df\n    .group_by(\"lab_id\")\n    .len()\n    .sort(\"len\", descending=True)\n)\nx = labid_counts[\"lab_id\"].to_list()\ny = labid_counts[\"len\"].to_list()\n\nplt.figure(figsize=(12, 5))\nbars = plt.bar(x, y)\nfor bar, count in zip(bars, y):\n    plt.text(\n        bar.get_x() + bar.get_width() / 2, \n        bar.get_height(),                  \n        str(count),                        \n        ha=\"center\", va=\"bottom\", fontsize=8, rotation=0\n    )\n\nplt.xticks(rotation=45)\nplt.xlabel(\"lab_id\")\nplt.ylabel(\"count\")\nplt.title(\"Lab ID count\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:29.008035Z","iopub.execute_input":"2025-09-20T08:23:29.008530Z","iopub.status.idle":"2025-09-20T08:23:29.570411Z","shell.execute_reply.started":"2025-09-20T08:23:29.008493Z","shell.execute_reply":"2025-09-20T08:23:29.569307Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- Almost all Lab IDs are MAB22_keypoints and MABe22_movies.","metadata":{}},{"cell_type":"code","source":"# =================\n# video_id\n# =================\nunique_videoid = list(train_df[\"video_id\"].unique())\n\nsep(\"video_id info\")\nprint(f\"n_unique (video_id): {len(unique_videoid)}/{len(train_df)}\")\nprint(unique_videoid[:3])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:33.770788Z","iopub.execute_input":"2025-09-20T08:23:33.771206Z","iopub.status.idle":"2025-09-20T08:23:33.785242Z","shell.execute_reply.started":"2025-09-20T08:23:33.771180Z","shell.execute_reply":"2025-09-20T08:23:33.783912Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ==================\n# mouse1_strain\n# ==================","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:34.364729Z","iopub.execute_input":"2025-09-20T08:23:34.365069Z","iopub.status.idle":"2025-09-20T08:23:34.369441Z","shell.execute_reply.started":"2025-09-20T08:23:34.365045Z","shell.execute_reply":"2025-09-20T08:23:34.368378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# [train/test]_tracking/ The feature data.\n---\n- video_frame\n- mouse_id\n- bodypart - The tracked body part. Different labs may track different body parts\n- [x/y] - The body part's X/Y coordinate position in pixels.","metadata":{}},{"cell_type":"code","source":"train_tracking_dirs = glob_walk(PATHS.train_tracking_dir, \"*\")\nprint(f\"n_train_tracking_dirs/n_unique_labid): {len(train_tracking_dirs)}/{len(unique_labid)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:23:51.564126Z","iopub.execute_input":"2025-09-20T08:23:51.564416Z","iopub.status.idle":"2025-09-20T08:23:51.580947Z","shell.execute_reply.started":"2025-09-20T08:23:51.564396Z","shell.execute_reply":"2025-09-20T08:23:51.579773Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example\nsample = pl.read_parquet(\"/kaggle/input/MABe-mouse-behavior-detection/train_tracking/MABe22_movies/1000942438.parquet\")\nsample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:38:38.535838Z","iopub.execute_input":"2025-09-20T08:38:38.536240Z","iopub.status.idle":"2025-09-20T08:38:38.557295Z","shell.execute_reply.started":"2025-09-20T08:38:38.536210Z","shell.execute_reply":"2025-09-20T08:38:38.556286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample.filter(pl.col(\"video_frame\")==0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T09:02:19.866407Z","iopub.execute_input":"2025-09-20T09:02:19.866921Z","iopub.status.idle":"2025-09-20T09:02:19.884259Z","shell.execute_reply.started":"2025-09-20T09:02:19.866894Z","shell.execute_reply":"2025-09-20T09:02:19.883251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_label_track_dict = defaultdict()\n\nfor _dir in train_tracking_dirs:\n    label_id = str(os.path.basename(_dir))\n    _paths = glob_walk(_dir, \"*.parquet\")\n    n_label_track_dict[label_id] = len(_paths)\n\nsorted_dict = dict(sorted(n_label_track_dict.items(), key=lambda x: x[1], reverse=True))\nx = list(sorted_dict.keys())\ny = list(sorted_dict.values())\n\nplt.figure(figsize=(12, 5))\nbars = plt.bar(x, y)\nfor bar, count in zip(bars, y):\n    plt.text(\n        bar.get_x() + bar.get_width() / 2,\n        bar.get_height(),\n        str(count),\n        ha=\"center\", va=\"bottom\", fontsize=8\n    )\n\nplt.xticks(rotation=45)\nplt.xlabel(\"label_id\")\nplt.ylabel(\"count\")\nplt.title(\"Count per label_id\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:54:47.140121Z","iopub.execute_input":"2025-09-20T08:54:47.140493Z","iopub.status.idle":"2025-09-20T08:54:47.662153Z","shell.execute_reply.started":"2025-09-20T08:54:47.140463Z","shell.execute_reply":"2025-09-20T08:54:47.660893Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tracking_cols = list(sample.columns)\nprint(f\"tracking_cols: {tracking_cols}\")\n\nlength_dict_tracking = defaultdict(list)\nfor _dir in train_tracking_dirs:\n    label_id = str(os.path.basename(_dir))\n    _paths = glob_walk(_dir, \"*.parquet\")\n    for _path in tqdm(_paths, desc=label_id, total=len(_paths)):\n        _df = pl.read_parquet(_path)\n        assert list(_df.columns) == tracking_cols, f\"Columns mismatch in {_path}\"\n        length_dict_tracking[label_id].append(len(_df))\n\n# Convert to DataFrame（label_id, length）\nrows = [{\"label_id\": k, \"length\": L} \n        for k, lst in length_dict_tracking.items() \n        for L in lst]\nlen_df = pd.DataFrame(rows)\n\n# Boxplot\ndata = [len_df.loc[len_df[\"label_id\"] == lab, \"length\"].values for lab in x]\n\nplt.figure(figsize=(14, 6))\nplt.boxplot(\n    data,\n    labels=x, \n    showfliers=False, \n    vert=True\n)\nplt.xticks(rotation=45, ha=\"right\")\nplt.xlabel(\"label_id\")\nplt.ylabel(\"length\")\nplt.title(\"Sequence length distribution per label_id\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:57:46.838754Z","iopub.execute_input":"2025-09-20T08:57:46.839111Z","iopub.status.idle":"2025-09-20T08:57:47.314105Z","shell.execute_reply.started":"2025-09-20T08:57:46.839086Z","shell.execute_reply":"2025-09-20T08:57:47.312742Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# MABe22_keypoints\nprint(len(data[0]))\ndata[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:59:24.031035Z","iopub.execute_input":"2025-09-20T08:59:24.031409Z","iopub.status.idle":"2025-09-20T08:59:24.040073Z","shell.execute_reply.started":"2025-09-20T08:59:24.031383Z","shell.execute_reply":"2025-09-20T08:59:24.038940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# MABe22_movies\nprint(len(data[1]))\ndata[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:59:43.973555Z","iopub.execute_input":"2025-09-20T08:59:43.973918Z","iopub.status.idle":"2025-09-20T08:59:43.981374Z","shell.execute_reply.started":"2025-09-20T08:59:43.973893Z","shell.execute_reply":"2025-09-20T08:59:43.979913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# AdaptableShall\nprint(len(data[-6]))\ndata[-6]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T09:01:23.084363Z","iopub.execute_input":"2025-09-20T09:01:23.084715Z","iopub.status.idle":"2025-09-20T09:01:23.093313Z","shell.execute_reply.started":"2025-09-20T09:01:23.084690Z","shell.execute_reply":"2025-09-20T09:01:23.092016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# VoisterousParrot\nprint(len(data[-1]))\ndata[-1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T09:03:15.982419Z","iopub.execute_input":"2025-09-20T09:03:15.982835Z","iopub.status.idle":"2025-09-20T09:03:15.991306Z","shell.execute_reply.started":"2025-09-20T09:03:15.982809Z","shell.execute_reply":"2025-09-20T09:03:15.989327Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# train_annotation/ Train labels.\n---\n- agent_id - ID of the mouse performing the behavior.\n- target_id - ID of the mouse targeted by the behavior. Some behaviors, such as a mouse grooming itself, have the same agent and target ID.\n- action - What behavior is occurring (e.g., grooming, chasing). Different labs annotated different behaviors.\n- [start/stop]_frame - The first/last frame of the action.","metadata":{}},{"cell_type":"code","source":"train_annotation_dirs = glob_walk(PATHS.train_annotation_dir, \"*\")\nprint(f\"n_train_annotation_dirs/n_unique_labid): {len(train_annotation_dirs)}/{len(unique_labid)}\")\nprint()\nsep(\"not involved labelid in train_annotation_dirs\")\ntrain_annotation_labelid = [str(os.path.basename(col)) for col in train_annotation_dirs]\nprint([col for col in unique_labid if col not in train_annotation_labelid])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:29:03.359377Z","iopub.execute_input":"2025-09-20T08:29:03.359692Z","iopub.status.idle":"2025-09-20T08:29:03.370129Z","shell.execute_reply.started":"2025-09-20T08:29:03.359671Z","shell.execute_reply":"2025-09-20T08:29:03.368769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Example\nsample = pl.read_parquet(\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/CalMS21_supplemental/1006083669.parquet\")\nsample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T08:30:37.253300Z","iopub.execute_input":"2025-09-20T08:30:37.253682Z","iopub.status.idle":"2025-09-20T08:30:37.270999Z","shell.execute_reply.started":"2025-09-20T08:30:37.253646Z","shell.execute_reply":"2025-09-20T08:30:37.269839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_label_anno_dict = defaultdict()\n\nfor _dir in train_annotation_dirs:\n    label_id = str(os.path.basename(_dir))\n    _paths = glob_walk(_dir, \"*.parquet\")\n    n_label_anno_dict[label_id] = len(_paths)\n\nsorted_dict = dict(sorted(n_label_anno_dict.items(), key=lambda x: x[1], reverse=True))\nx = list(sorted_dict.keys())\ny = list(sorted_dict.values())\n\nplt.figure(figsize=(12, 5))\nbars = plt.bar(x, y)\nfor bar, count in zip(bars, y):\n    plt.text(\n        bar.get_x() + bar.get_width() / 2,\n        bar.get_height(),\n        str(count),\n        ha=\"center\", va=\"bottom\", fontsize=8\n    )\n\nplt.xticks(rotation=45)\nplt.xlabel(\"label_id\")\nplt.ylabel(\"count\")\nplt.title(\"Count per label_id\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T09:03:27.245678Z","iopub.execute_input":"2025-09-20T09:03:27.246019Z","iopub.status.idle":"2025-09-20T09:03:27.894339Z","shell.execute_reply.started":"2025-09-20T09:03:27.245994Z","shell.execute_reply":"2025-09-20T09:03:27.893064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"anno_cols = list(_df.columns)\nprint(f\"annotation_cols: {anno_cols}\")\n\nlength_dict_anno = defaultdict(list)\nfor _dir in train_annotation_dirs:\n    label_id = str(os.path.basename(_dir))\n    _paths = glob_walk(_dir, \"*.parquet\")\n    for i, _path in enumerate(tqdm(_paths, desc=label_id, total=len(_paths))):\n        _df = pl.read_parquet(_path)\n        length_dict_anno[label_id].append(len(_df))\n\n# Convert to DataFrame（label_id, length）\nrows = [{\"label_id\": k, \"length\": L} \n        for k, lst in length_dict_anno.items() \n        for L in lst]\nlen_df = pd.DataFrame(rows)\n\n# Boxplot\ndata = [len_df.loc[len_df[\"label_id\"] == lab, \"length\"].values for lab in x]\n\nplt.figure(figsize=(14, 6))\nplt.boxplot(\n    data,\n    labels=x, \n    showfliers=False, \n    vert=True\n)\nplt.xticks(rotation=45, ha=\"right\")\nplt.xlabel(\"label_id\")\nplt.ylabel(\"length\")\nplt.title(\"Sequence length distribution per label_id\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-20T09:07:30.107268Z","iopub.execute_input":"2025-09-20T09:07:30.108499Z","iopub.status.idle":"2025-09-20T09:07:37.618554Z","shell.execute_reply.started":"2025-09-20T09:07:30.108458Z","shell.execute_reply":"2025-09-20T09:07:37.617519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}