{"cells":[{"cell_type":"markdown","id":"60bd2432","metadata":{"papermill":{"duration":0.00613,"end_time":"2025-11-04T09:10:16.721102","exception":false,"start_time":"2025-11-04T09:10:16.714972","status":"completed"},"tags":[]},"source":"# Imports and configs"},{"cell_type":"code","execution_count":1,"id":"64949602","metadata":{"_kg_hide-input":true,"execution":{"iopub.execute_input":"2025-11-04T09:10:16.735782Z","iopub.status.busy":"2025-11-04T09:10:16.735457Z","iopub.status.idle":"2025-11-04T09:10:19.819247Z","shell.execute_reply":"2025-11-04T09:10:19.818059Z"},"papermill":{"duration":3.094214,"end_time":"2025-11-04T09:10:19.821264","exception":false,"start_time":"2025-11-04T09:10:16.72705","status":"completed"},"tags":[]},"outputs":[],"source":"\"\"\"F Beta customized for the data format of the MABe challenge.\"\"\"\n\nimport json\n\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\n\n\nclass HostVisibleError(Exception):\n pass\n\n\ndef single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n label_frames: defaultdict[str, set[int]] = defaultdict(set)\n prediction_frames: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_solution.to_dicts():\n label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n for video in lab_solution['video_id'].unique():\n active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first() # ty: ignore\n active_labels: set[str] = set(json.loads(active_labels))\n predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_submission.filter(pl.col('video_id') == video).to_dicts():\n # Since the labels are sparse, we can't evaluate prediction keys not in the active labels.\n if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n continue\n\n new_frames = set(range(row['start_frame'], row['stop_frame']))\n # Ignore truly redundant predictions.\n new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n # A single agent can have multiple targets per frame (ex: evading all other mice) but only one action per target per frame.\n raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n prediction_frames[row['prediction_key']].update(new_frames)\n predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n tps = defaultdict(int)\n fns = defaultdict(int)\n fps = defaultdict(int)\n for key, pred_frames in prediction_frames.items():\n action = key.split('_')[-1]\n matched_label_frames = label_frames[key]\n tps[action] += len(pred_frames.intersection(matched_label_frames))\n fns[action] += len(matched_label_frames.difference(pred_frames))\n fps[action] += len(pred_frames.difference(matched_label_frames))\n\n distinct_actions = set()\n for key, frames in label_frames.items():\n action = key.split('_')[-1]\n distinct_actions.add(action)\n if key not in prediction_frames:\n fns[action] += len(frames)\n\n action_f1s = []\n for action in distinct_actions:\n if tps[action] + fns[action] + fps[action] == 0:\n action_f1s.append(0)\n else:\n action_f1s.append((1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action]))\n return sum(action_f1s) / len(action_f1s)\n\n\ndef mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n \"\"\"\n Doctests:\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 0, 'stop_frame': 10}, # Wrong action\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.500000000000'\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.250000000000'\n\n >>> # Overlapping solution events, one prediction matching both.\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 10, 'stop_frame': 20, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 20},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 30, 'stop_frame': 40, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 40},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.6666666666666666\n \"\"\"\n if len(solution) == 0 or len(submission) == 0:\n raise ValueError('Missing solution or submission data')\n\n expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n for col in expected_cols:\n if col not in solution.columns:\n raise ValueError(f'Solution is missing column {col}')\n if col not in submission.columns:\n raise ValueError(f'Submission is missing column {col}')\n\n solution: pl.DataFrame = pl.DataFrame(solution)\n submission: pl.DataFrame = pl.DataFrame(submission)\n assert (solution['start_frame'] <= solution['stop_frame']).all()\n assert (submission['start_frame'] <= submission['stop_frame']).all()\n solution_videos = set(solution['video_id'].unique())\n # Need to align based on video IDs as we can't rely on the row IDs for handling public/private splits.\n submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n solution = solution.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('label_key'),\n )\n submission = submission.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('prediction_key'),\n )\n\n lab_scores = []\n for lab in solution['lab_id'].unique():\n lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n lab_videos = set(lab_solution['video_id'].unique())\n lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n lab_scores.append(single_lab_f1(lab_solution, lab_submission, beta=beta))\n\n return sum(lab_scores) / len(lab_scores)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n \"\"\"\n F1 score for the MABe Challenge\n \"\"\"\n solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n return mouse_fbeta(solution, submission, beta=beta)"},{"cell_type":"code","execution_count":2,"id":"6d4e1578","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:19.835416Z","iopub.status.busy":"2025-11-04T09:10:19.83486Z","iopub.status.idle":"2025-11-04T09:10:22.400121Z","shell.execute_reply":"2025-11-04T09:10:22.399119Z"},"papermill":{"duration":2.5743,"end_time":"2025-11-04T09:10:22.40198","exception":false,"start_time":"2025-11-04T09:10:19.82768","status":"completed"},"tags":[]},"outputs":[],"source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.base import clone\nfrom xgboost import XGBClassifier\nfrom tqdm.notebook import tqdm\nfrom koolbox import Trainer\n\nimport numpy as np\nimport itertools\nimport warnings\nimport optuna\nimport joblib\nimport glob\nimport gc\nimport os\nimport subprocess\nimport json\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')"},{"cell_type":"code","execution_count":3,"id":"fa3a17c6","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.416784Z","iopub.status.busy":"2025-11-04T09:10:22.416288Z","iopub.status.idle":"2025-11-04T09:10:22.422997Z","shell.execute_reply":"2025-11-04T09:10:22.422048Z"},"papermill":{"duration":0.015603,"end_time":"2025-11-04T09:10:22.424571","exception":false,"start_time":"2025-11-04T09:10:22.408968","status":"completed"},"tags":[]},"outputs":[],"source":"class CFG:\n \n # Paths\n DATA_DIR = \"/mnt/lfs0/Database/MABe-mouse-behavior-detection\"\n\n TRAIN_PATH = os.path.join(DATA_DIR, \"train.csv\")\n TEST_PATH = os.path.join(DATA_DIR, \"test.csv\")\n\n TRAIN_TRACKING_PATH = os.path.join(DATA_DIR, \"train_tracking\")\n TEST_TRACKING_PATH = os.path.join(DATA_DIR, \"test_tracking\")\n TRAIN_ANNOTATION_PATH = os.path.join(DATA_DIR, \"train_annotation\")\n\n # Backward compatibility with older attribute names\n train_path = TRAIN_PATH\n test_path = TEST_PATH\n train_tracking_path = TRAIN_TRACKING_PATH\n test_tracking_path = TEST_TRACKING_PATH\n train_annotation_path = TRAIN_ANNOTATION_PATH\n\n # Models will be stored under:\n # /mnt/lfs0/Database/MABe-mouse-behavior-detection/models/xgb\n model_path = os.path.join(DATA_DIR, \"models\")\n model_name = \"xgb\"\n\n \n # Cross-validation\n n_splits = 3\n cv = StratifiedGroupKFold(n_splits=n_splits)\n \n # Ablation switches\n group_by_lab = False # True: lab_id / False: video_id grouping\n use_metadata = True # True: pose + meta (mouse and arena)\n use_scale_pos_weight = False # True: scale postive actions\n\n # Hardware configuration\n use_gpu = True # set to False to force CPU training\n gpu_id = 1\n\n @staticmethod\n def _gpu_is_available(device_id: int | None = None) -> bool:\n \"\"\"Light-weight check if a CUDA GPU is visible and usable.\"\"\"\n try:\n env = os.environ.copy()\n visible = env.get(\"CUDA_VISIBLE_DEVICES\")\n if visible is not None and visible.strip() in (\"\", \"-1\"):\n return False\n if device_id is not None and \"CUDA_VISIBLE_DEVICES\" not in env:\n env[\"CUDA_VISIBLE_DEVICES\"] = str(device_id)\n\n subprocess.run(\n [\"nvidia-smi\", \"-L\"],\n stdout=subprocess.DEVNULL,\n stderr=subprocess.DEVNULL,\n check=True,\n timeout=3,\n env=env,\n )\n return True\n except Exception:\n return False\n\n _gpu_enabled = use_gpu and _gpu_is_available(gpu_id)\n\n # Base XGBoost parameters (per-action class weighting will be added later)\n _xgb_params = dict(\n verbosity=0,\n random_state=42,\n n_estimators=250,\n learning_rate=0.08,\n max_depth=6,\n min_child_weight=5,\n subsample=0.8,\n colsample_bytree=0.8,\n objective=\"binary:logistic\",\n eval_metric=\"logloss\",\n )\n\n if _gpu_enabled:\n _xgb_params.update(\n tree_method=\"hist\",\n device=f\"cuda:{gpu_id}\",\n )\n else:\n if use_gpu:\n warnings.warn(\"GPU requested but not available. Falling back to CPU.\")\n _xgb_params.update(\n tree_method=\"hist\",\n predictor=\"auto\",\n device=\"cpu\",\n )\n\n # Base estimator; we will clone it and override `scale_pos_weight` per action\n model = XGBClassifier(**_xgb_params)"},{"cell_type":"markdown","id":"d3bf42a2","metadata":{"papermill":{"duration":0.006723,"end_time":"2025-11-04T09:10:22.437531","exception":false,"start_time":"2025-11-04T09:10:22.430808","status":"completed"},"tags":[]},"source":"# Data loading and preprocessing"},{"cell_type":"code","execution_count":4,"id":"3a491a32","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.452713Z","iopub.status.busy":"2025-11-04T09:10:22.452002Z","iopub.status.idle":"2025-11-04T09:10:22.6393Z","shell.execute_reply":"2025-11-04T09:10:22.638399Z"},"papermill":{"duration":0.197178,"end_time":"2025-11-04T09:10:22.64103","exception":false,"start_time":"2025-11-04T09:10:22.443852","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Arena meta shape: (8790, 44)\n","Mouse meta shape: (25535, 23)\n"]}],"source":"# Data loading\ntrain = pd.read_csv(CFG.TRAIN_PATH)\ntest = pd.read_csv(CFG.TEST_PATH)\n\n# Ensure integer type for video_id\ntrain[\"video_id\"] = train[\"video_id\"].astype(int)\ntest[\"video_id\"] = test[\"video_id\"].astype(int)\n\n# Number of mice in each video\nmouse_strain_cols = [f\"mouse{i}_strain\" for i in range(1, 5)]\nfor df in (train, test):\n df[\"n_mice\"] = 4 - df[mouse_strain_cols].isna().sum(axis=1)\n\n# Exclude MABe22 for training/validation, as in the original notebook\ntrain_without_mabe22 = train.query(\"~lab_id.str.startswith('MABe22_')\")\n\n\ndef build_arena_and_mouse_metadata(\n train_df: pd.DataFrame,\n test_df: pd.DataFrame,\n) -> tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"Build compact arena- and mouse-level metadata to keep feature width small.\"\"\"\n\n # ------------------------- arena-level metadata -------------------------\n arena_cols = [\n \"video_id\",\n \"lab_id\",\n \"frames_per_second\",\n \"video_duration_sec\",\n \"pix_per_cm_approx\",\n \"arena_width_cm\",\n \"arena_height_cm\",\n \"arena_shape\",\n \"arena_type\",\n \"tracking_method\",\n \"n_mice\",\n ]\n\n arena_all = pd.concat(\n [train_df[arena_cols], test_df[arena_cols]],\n axis=0,\n ignore_index=True,\n )\n arena_all = arena_all.drop_duplicates(\"video_id\").set_index(\"video_id\")\n\n arena_cat_cols = [\"lab_id\", \"arena_shape\", \"arena_type\", \"tracking_method\"]\n arena_meta = pd.get_dummies(\n arena_all,\n columns=arena_cat_cols,\n dummy_na=True,\n )\n\n # ------------------------- mouse-level metadata -------------------------\n def _stack_mouse_metadata(df: pd.DataFrame) -> pd.DataFrame:\n records = []\n for m in range(1, 5):\n cols = [\n \"video_id\",\n f\"mouse{m}_strain\",\n f\"mouse{m}_color\",\n f\"mouse{m}_sex\",\n f\"mouse{m}_age\",\n f\"mouse{m}_condition\",\n ]\n sub = df[cols].copy()\n sub = sub.rename(\n columns={\n f\"mouse{m}_strain\": \"strain\",\n f\"mouse{m}_color\": \"color\",\n f\"mouse{m}_sex\": \"sex\",\n f\"mouse{m}_age\": \"age\",\n f\"mouse{m}_condition\": \"condition\",\n }\n )\n sub[\"agent_id\"] = f\"mouse{m}\"\n sub = sub[~sub[\"strain\"].isna()]\n records.append(sub)\n return pd.concat(records, axis=0, ignore_index=True)\n\n mouse_all = pd.concat(\n [_stack_mouse_metadata(train_df), _stack_mouse_metadata(test_df)],\n axis=0,\n ignore_index=True,\n )\n mouse_all = mouse_all.drop_duplicates([\"video_id\", \"agent_id\"]).set_index(\n [\"video_id\", \"agent_id\"]\n )\n\n dfm = mouse_all.copy()\n\n # 1) Low-cardinality one-hot: strain, color, sex\n for col in [\"strain\", \"color\", \"sex\"]:\n dummies = pd.get_dummies(dfm[col], prefix=col, dummy_na=True)\n dfm = pd.concat([dfm.drop(columns=[col]), dummies], axis=1)\n\n # 2) Age: approximate numeric value\n age_num = dfm[\"age\"].astype(str).str.extract(r\"(\\d+)\", expand=False)\n age_num = pd.to_numeric(age_num, errors=\"coerce\")\n dfm[\"age_weeks\"] = age_num\n dfm.drop(columns=[\"age\"], inplace=True)\n\n # 3) Condition: compact encoding instead of full one-hot\n cond_series = dfm[\"condition\"]\n cond_lower = cond_series.astype(str).str.lower()\n dfm[\"cond_has_device\"] = cond_lower.str.contains(\"device\").astype(\"int8\")\n dfm[\"cond_has_cable\"] = cond_lower.str.contains(\"cable\").astype(\"int8\")\n dfm[\"cond_is_na\"] = cond_series.isna().astype(\"int8\")\n dfm[\"cond_code\"] = cond_series.astype(\"category\").cat.codes.astype(\"int16\")\n dfm.drop(columns=[\"condition\"], inplace=True)\n\n return arena_meta, dfm\n\n\narena_meta, mouse_meta = build_arena_and_mouse_metadata(train, test)\nprint(f\"Arena meta shape: {arena_meta.shape}\")\nprint(f\"Mouse meta shape: {mouse_meta.shape}\")\n\n# List of body-parts-tracked configurations\nbody_parts_tracked_list = list(np.unique(train.body_parts_tracked))\n"},{"cell_type":"markdown","id":"c976df8b","metadata":{"papermill":{"duration":0.006286,"end_time":"2025-11-04T09:10:22.677292","exception":false,"start_time":"2025-11-04T09:10:22.671006","status":"completed"},"tags":[]},"source":"## Creating solution data"},{"cell_type":"code","execution_count":5,"id":"f2f7c688","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.691831Z","iopub.status.busy":"2025-11-04T09:10:22.691511Z","iopub.status.idle":"2025-11-04T09:10:22.699481Z","shell.execute_reply":"2025-11-04T09:10:22.698303Z"},"papermill":{"duration":0.016962,"end_time":"2025-11-04T09:10:22.70113","exception":false,"start_time":"2025-11-04T09:10:22.684168","status":"completed"},"tags":[]},"outputs":[],"source":"def create_solution_df(dataset):\n solution = []\n for _, row in tqdm(dataset.iterrows(), total=len(dataset)):\n \n lab_id = row['lab_id']\n if lab_id.startswith('MABe22'): \n continue\n \n video_id = row['video_id']\n path = f\"{CFG.train_annotation_path}/{lab_id}/{video_id}.parquet\"\n try:\n annot = pd.read_parquet(path)\n except FileNotFoundError:\n continue\n \n annot['lab_id'] = lab_id\n annot['video_id'] = video_id\n annot['behaviors_labeled'] = row['behaviors_labeled']\n annot['target_id'] = np.where(annot.target_id != annot.agent_id, annot['target_id'].apply(lambda s: f\"mouse{s}\"), 'self')\n annot['agent_id'] = annot['agent_id'].apply(lambda s: f\"mouse{s}\")\n solution.append(annot)\n \n solution = pd.concat(solution)\n \n return solution\n\n"},{"cell_type":"markdown","id":"92da7512","metadata":{"papermill":{"duration":0.007009,"end_time":"2025-11-04T09:10:22.716462","exception":false,"start_time":"2025-11-04T09:10:22.709453","status":"completed"},"tags":[]},"source":"## Data generator"},{"cell_type":"code","execution_count":6,"id":"229715c6","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.733735Z","iopub.status.busy":"2025-11-04T09:10:22.733439Z","iopub.status.idle":"2025-11-04T09:10:22.752907Z","shell.execute_reply":"2025-11-04T09:10:22.751532Z"},"papermill":{"duration":0.032299,"end_time":"2025-11-04T09:10:22.755125","exception":false,"start_time":"2025-11-04T09:10:22.722826","status":"completed"},"tags":[]},"outputs":[],"source":"drop_body_parts = [\n \"headpiece_bottombackleft\",\n \"headpiece_bottombackright\",\n \"headpiece_bottomfrontleft\",\n \"headpiece_bottomfrontright\",\n \"headpiece_topbackleft\",\n \"headpiece_topbackright\",\n \"headpiece_topfrontleft\",\n \"headpiece_topfrontright\",\n \"spine_1\",\n \"spine_2\",\n \"tail_middle_1\",\n \"tail_middle_2\",\n \"tail_midpoint\",\n]\n\n\ndef generate_mouse_data(\n dataset: pd.DataFrame,\n traintest: str,\n traintest_directory: str | None = None,\n generate_single: bool = True,\n generate_pair: bool = True,\n):\n \"\"\"Yield (switch, data, meta, label_or_actions) for single or pair mice.\"\"\"\n if traintest_directory is None:\n if traintest == \"train\":\n traintest_directory = CFG.TRAIN_TRACKING_PATH\n else:\n traintest_directory = CFG.TEST_TRACKING_PATH\n\n for _, row in dataset.iterrows():\n lab_id = row.lab_id\n if lab_id.startswith(\"MABe22\") or type(row.behaviors_labeled) != str:\n continue\n\n video_id = row.video_id\n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n\n vid = pd.read_parquet(path)\n if len(np.unique(vid.bodypart)) > 5:\n vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n pvid = vid.pivot(\n columns=[\"mouse_id\", \"bodypart\"],\n index=\"video_frame\",\n values=[\"x\", \"y\"],\n )\n\n del vid\n gc.collect()\n\n # Move levels to (bodypart, coord, mouse_id) to match transform_* functions\n pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n # Convert pixels to centimeters\n pvid /= row.pix_per_cm_approx\n\n vid_behaviors = json.loads(row.behaviors_labeled)\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(\",\") for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(\n vid_behaviors, columns=[\"agent\", \"target\", \"action\"]\n )\n\n if traintest == \"train\":\n try:\n annot = pd.read_parquet(\n path.replace(\"train_tracking\", \"train_annotation\")\n )\n except FileNotFoundError:\n continue\n\n # ----------------------------- single-mouse -----------------------------\n if generate_single:\n vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n try:\n mouse_id = int(mouse_id_str[-1])\n vid_agent_actions = np.unique(\n vid_behaviors_subset.query(\"agent == @mouse_id_str\").action\n )\n single_mouse = pvid.loc[:, mouse_id]\n assert len(single_mouse) == len(pvid)\n single_mouse_meta = pd.DataFrame(\n {\n \"lab_id\": lab_id,\n \"video_id\": video_id,\n \"agent_id\": mouse_id_str,\n \"target_id\": \"self\",\n \"video_frame\": single_mouse.index,\n }\n )\n if traintest == \"train\":\n single_mouse_label = pd.DataFrame(\n 0.0,\n columns=vid_agent_actions,\n index=single_mouse.index,\n )\n annot_subset = annot.query(\n \"(agent_id == @mouse_id) & (target_id == @mouse_id)\"\n )\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n single_mouse_label.loc[\n annot_row[\"start_frame\"] : annot_row[\"stop_frame\"],\n annot_row.action,\n ] = 1.0\n yield \"single\", single_mouse, single_mouse_meta, single_mouse_label\n else:\n yield \"single\", single_mouse, single_mouse_meta, vid_agent_actions\n except KeyError:\n # This mouse might not exist in this video\n pass\n\n # ----------------------------- mouse pairs ------------------------------\n if generate_pair:\n vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n if len(vid_behaviors_subset) > 0:\n for agent, target in itertools.permutations(\n np.unique(pvid.columns.get_level_values(\"mouse_id\")), 2\n ):\n agent_str = f\"mouse{agent}\"\n target_str = f\"mouse{target}\"\n vid_agent_actions = np.unique(\n vid_behaviors_subset.query(\n \"(agent == @agent_str) & (target == @target_str)\"\n ).action\n )\n mouse_pair = pd.concat(\n [pvid[agent], pvid[target]], axis=1, keys=[\"A\", \"B\"]\n )\n assert len(mouse_pair) == len(pvid)\n mouse_pair_meta = pd.DataFrame(\n {\n \"lab_id\": lab_id,\n \"video_id\": video_id,\n \"agent_id\": agent_str,\n \"target_id\": target_str,\n \"video_frame\": mouse_pair.index,\n }\n )\n if traintest == \"train\":\n mouse_pair_label = pd.DataFrame(\n 0.0,\n columns=vid_agent_actions,\n index=mouse_pair.index,\n )\n annot_subset = annot.query(\n \"(agent_id == @agent) & (target_id == @target)\"\n )\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n mouse_pair_label.loc[\n annot_row[\"start_frame\"] : annot_row[\"stop_frame\"],\n annot_row.action,\n ] = 1.0\n yield \"pair\", mouse_pair, mouse_pair_meta, mouse_pair_label\n else:\n yield \"pair\", mouse_pair, mouse_pair_meta, vid_agent_actions\n"},{"cell_type":"markdown","id":"0c5b2ecf","metadata":{"papermill":{"duration":0.005992,"end_time":"2025-11-04T09:10:22.767349","exception":false,"start_time":"2025-11-04T09:10:22.761357","status":"completed"},"tags":[]},"source":"## Transforming coordinates"},{"cell_type":"code","execution_count":7,"id":"9c3d4c1e","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.783605Z","iopub.status.busy":"2025-11-04T09:10:22.78229Z","iopub.status.idle":"2025-11-04T09:10:22.809779Z","shell.execute_reply":"2025-11-04T09:10:22.808628Z"},"papermill":{"duration":0.037543,"end_time":"2025-11-04T09:10:22.811464","exception":false,"start_time":"2025-11-04T09:10:22.773921","status":"completed"},"tags":[]},"outputs":[],"source":"def safe_rolling(series, window, func, min_periods=None):\n if min_periods is None:\n min_periods = max(1, window // 4)\n return series.rolling(window, min_periods=min_periods, center=True).apply(func, raw=True)\n\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n if n_frames_at_30fps == 0:\n return 0\n s = 1 if n_frames_at_30fps > 0 else -1\n mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n return s * mag\n\ndef _fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n if 'frames_per_second' in meta_df.columns and pd.notnull(meta_df['frames_per_second']).any():\n return float(meta_df['frames_per_second'].iloc[0])\n vid = meta_df['video_id'].iloc[0]\n return float(fallback_lookup.get(vid, default_fps))\n\ndef add_curvature_features(X, center_x, center_y, fps):\n vel_x = center_x.diff()\n vel_y = center_y.diff()\n acc_x = vel_x.diff()\n acc_y = vel_y.diff()\n\n cross_prod = vel_x * acc_y - vel_y * acc_x\n vel_mag = np.sqrt(vel_x**2 + vel_y**2)\n curvature = np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n\n for w in [25, 50, 75]:\n ws = _scale(w, fps)\n X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 5)).mean()\n\n angle = np.arctan2(vel_y, vel_x)\n angle_change = np.abs(angle.diff())\n w = 30\n ws = _scale(w, fps)\n X[f'turn_rate_{w}'] = angle_change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n\n return X\n\ndef add_multiscale_features(X, center_x, center_y, fps):\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n\n scales = [20, 40, 60, 80]\n for scale in scales:\n ws = _scale(scale, fps)\n if len(speed) >= ws:\n X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n\n if len(scales) >= 2 and f'sp_m{scales[0]}' in X.columns and f'sp_m{scales[-1]}' in X.columns:\n X['sp_ratio'] = X[f'sp_m{scales[0]}'] / (X[f'sp_m{scales[-1]}'] + 1e-6)\n\n return X\n\ndef add_state_features(X, center_x, center_y, fps):\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n w_ma = _scale(15, fps)\n speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n\n try:\n bins = [-np.inf, 0.5 * fps, 2.0 * fps, 5.0 * fps, np.inf]\n speed_states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n\n for window in [20, 40, 60, 80]:\n ws = _scale(window, fps)\n if len(speed_states) >= ws:\n for state in [0, 1, 2, 3]:\n X[f's{state}_{window}'] = (\n (speed_states == state).astype(float)\n .rolling(ws, min_periods=max(1, ws // 5)).mean()\n )\n state_changes = (speed_states != speed_states.shift(1)).astype(float)\n X[f'trans_{window}'] = state_changes.rolling(ws, min_periods=max(1, ws // 5)).sum()\n except Exception:\n pass\n\n return X\n\ndef add_longrange_features(X, center_x, center_y, fps):\n for window in [30, 60, 120]:\n ws = _scale(window, fps)\n if len(center_x) >= ws:\n X[f'x_ml{window}'] = center_x.rolling(ws, min_periods=max(5, ws // 6)).mean()\n X[f'y_ml{window}'] = center_y.rolling(ws, min_periods=max(5, ws // 6)).mean()\n\n for span in [30, 60, 120]:\n s = _scale(span, fps)\n X[f'x_e{span}'] = center_x.ewm(span=s, min_periods=1).mean()\n X[f'y_e{span}'] = center_y.ewm(span=s, min_periods=1).mean()\n\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps) # cm/s\n for window in [30, 60, 120]:\n ws = _scale(window, fps)\n if len(speed) >= ws:\n X[f'sp_pct{window}'] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n\n return X\n\ndef add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n if 'body_center' not in avail_A or 'body_center' not in avail_B:\n return X\n\n rel_x = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n rel_y = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n A_vx = mouse_pair['A']['body_center']['x'].diff()\n A_vy = mouse_pair['A']['body_center']['y'].diff()\n B_vx = mouse_pair['B']['body_center']['x'].diff()\n B_vy = mouse_pair['B']['body_center']['y'].diff()\n\n A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n for window in [30, 60]:\n ws = _scale(window, fps)\n X[f'A_ld{window}'] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n X[f'B_ld{window}'] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n approach = -rel_dist.diff()\n chase = approach * B_lead\n w = 30\n ws = _scale(w, fps)\n X[f'chase_{w}'] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n for window in [60, 120]:\n ws = _scale(window, fps)\n A_sp = np.sqrt(A_vx**2 + A_vy**2)\n B_sp = np.sqrt(B_vx**2 + B_vy**2)\n X[f'sp_cor{window}'] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n\n return X"},{"cell_type":"code","execution_count":8,"id":"2fd8a6e8","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.826828Z","iopub.status.busy":"2025-11-04T09:10:22.826512Z","iopub.status.idle":"2025-11-04T09:10:22.869098Z","shell.execute_reply":"2025-11-04T09:10:22.868261Z"},"papermill":{"duration":0.052913,"end_time":"2025-11-04T09:10:22.87102","exception":false,"start_time":"2025-11-04T09:10:22.818107","status":"completed"},"tags":[]},"outputs":[],"source":"def transform_single(single_mouse, body_parts_tracked, fps):\n available_body_parts = single_mouse.columns.get_level_values(0)\n\n X = pd.DataFrame({\n f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n if p1 in available_body_parts and p2 in available_body_parts\n })\n X = X.reindex(columns=[f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(body_parts_tracked, 2)], copy=False)\n\n if all(p in single_mouse.columns for p in ['ear_left', 'ear_right', 'tail_base']):\n lag = _scale(10, fps)\n shifted = single_mouse[['ear_left', 'ear_right', 'tail_base']].shift(lag)\n speeds = pd.DataFrame({\n 'sp_lf': np.square(single_mouse['ear_left'] - shifted['ear_left']).sum(axis=1, skipna=False),\n 'sp_rt': np.square(single_mouse['ear_right'] - shifted['ear_right']).sum(axis=1, skipna=False),\n 'sp_lf2': np.square(single_mouse['ear_left'] - shifted['tail_base']).sum(axis=1, skipna=False),\n 'sp_rt2': np.square(single_mouse['ear_right'] - shifted['tail_base']).sum(axis=1, skipna=False),\n })\n X = pd.concat([X, speeds], axis=1)\n\n if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n if all(p in available_body_parts for p in ['nose', 'body_center', 'tail_base']):\n v1 = single_mouse['nose'] - single_mouse['body_center']\n v2 = single_mouse['tail_base'] - single_mouse['body_center']\n X['body_ang'] = (v1['x'] * v2['x'] + v1['y'] * v2['y']) / (\n np.sqrt(v1['x']**2 + v1['y']**2) * np.sqrt(v2['x']**2 + v2['y']**2) + 1e-6)\n\n if 'body_center' in available_body_parts:\n cx = single_mouse['body_center']['x']\n cy = single_mouse['body_center']['y']\n\n for w in [5, 15, 30, 60]:\n ws = _scale(w, fps)\n roll = dict(min_periods=1, center=True)\n X[f'cx_m{w}'] = cx.rolling(ws, **roll).mean()\n X[f'cy_m{w}'] = cy.rolling(ws, **roll).mean()\n X[f'cx_s{w}'] = cx.rolling(ws, **roll).std()\n X[f'cy_s{w}'] = cy.rolling(ws, **roll).std()\n X[f'x_rng{w}'] = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n X[f'y_rng{w}'] = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n X[f'disp{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).sum()**2 +\n cy.diff().rolling(ws, min_periods=1).sum()**2)\n X[f'act{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).var() +\n cy.diff().rolling(ws, min_periods=1).var())\n\n X = add_curvature_features(X, cx, cy, fps)\n X = add_multiscale_features(X, cx, cy, fps)\n X = add_state_features(X, cx, cy, fps)\n X = add_longrange_features(X, cx, cy, fps)\n\n if all(p in available_body_parts for p in ['nose', 'tail_base']):\n nt_dist = np.sqrt((single_mouse['nose']['x'] - single_mouse['tail_base']['x'])**2 +\n (single_mouse['nose']['y'] - single_mouse['tail_base']['y'])**2)\n for lag in [10, 20, 40]:\n l = _scale(lag, fps)\n X[f'nt_lg{lag}'] = nt_dist.shift(l)\n X[f'nt_df{lag}'] = nt_dist - nt_dist.shift(l)\n\n if all(p in available_body_parts for p in ['ear_left', 'ear_right']):\n ear_d = np.sqrt((single_mouse['ear_left']['x'] - single_mouse['ear_right']['x'])**2 +\n (single_mouse['ear_left']['y'] - single_mouse['ear_right']['y'])**2)\n for off in [-30, -20, -10, 10, 20, 30]:\n o = _scale_signed(off, fps)\n X[f'ear_o{off}'] = ear_d.shift(-o)\n w = _scale(30, fps)\n X['ear_con'] = ear_d.rolling(w, min_periods=1, center=True).std() / \\\n (ear_d.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n return X.astype(np.float32, copy=False)\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n avail_A = mouse_pair['A'].columns.get_level_values(0)\n avail_B = mouse_pair['B'].columns.get_level_values(0)\n\n X = pd.DataFrame({\n f\"12+{p1}+{p2}\": np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n if p1 in avail_A and p2 in avail_B\n })\n X = X.reindex(columns=[f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(body_parts_tracked, repeat=2)], copy=False)\n\n if ('A', 'ear_left') in mouse_pair.columns and ('B', 'ear_left') in mouse_pair.columns:\n lag = _scale(10, fps)\n shA = mouse_pair['A']['ear_left'].shift(lag)\n shB = mouse_pair['B']['ear_left'].shift(lag)\n speeds = pd.DataFrame({\n 'sp_A': np.square(mouse_pair['A']['ear_left'] - shA).sum(axis=1, skipna=False),\n 'sp_AB': np.square(mouse_pair['A']['ear_left'] - shB).sum(axis=1, skipna=False),\n 'sp_B': np.square(mouse_pair['B']['ear_left'] - shB).sum(axis=1, skipna=False),\n })\n X = pd.concat([X, speeds], axis=1)\n\n if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n dir_A = mouse_pair['A']['nose'] - mouse_pair['A']['tail_base']\n dir_B = mouse_pair['B']['nose'] - mouse_pair['B']['tail_base']\n X['rel_ori'] = (dir_A['x'] * dir_B['x'] + dir_A['y'] * dir_B['y']) / (\n np.sqrt(dir_A['x']**2 + dir_A['y']**2) * np.sqrt(dir_B['x']**2 + dir_B['y']**2) + 1e-6)\n\n if all(p in avail_A for p in ['nose']) and all(p in avail_B for p in ['nose']):\n cur = np.square(mouse_pair['A']['nose'] - mouse_pair['B']['nose']).sum(axis=1, skipna=False)\n lag = _scale(10, fps)\n shA_n = mouse_pair['A']['nose'].shift(lag)\n shB_n = mouse_pair['B']['nose'].shift(lag)\n past = np.square(shA_n - shB_n).sum(axis=1, skipna=False)\n X['appr'] = cur - past\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n cd = np.sqrt((mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x'])**2 +\n (mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y'])**2)\n X['v_cls'] = (cd < 5.0).astype(float)\n X['cls'] = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n X['med'] = ((cd >= 15.0) & (cd < 30.0)).astype(float)\n X['far'] = (cd >= 30.0).astype(float)\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n cd_full = np.square(mouse_pair['A']['body_center'] - mouse_pair['B']['body_center']).sum(axis=1, skipna=False)\n\n for w in [5, 15, 30, 60]:\n ws = _scale(w, fps)\n roll = dict(min_periods=1, center=True)\n X[f'd_m{w}'] = cd_full.rolling(ws, **roll).mean()\n X[f'd_s{w}'] = cd_full.rolling(ws, **roll).std()\n X[f'd_mn{w}'] = cd_full.rolling(ws, **roll).min()\n X[f'd_mx{w}'] = cd_full.rolling(ws, **roll).max()\n\n d_var = cd_full.rolling(ws, **roll).var()\n X[f'int{w}'] = 1 / (1 + d_var)\n\n Axd = mouse_pair['A']['body_center']['x'].diff()\n Ayd = mouse_pair['A']['body_center']['y'].diff()\n Bxd = mouse_pair['B']['body_center']['x'].diff()\n Byd = mouse_pair['B']['body_center']['y'].diff()\n coord = Axd * Bxd + Ayd * Byd\n X[f'co_m{w}'] = coord.rolling(ws, **roll).mean()\n X[f'co_s{w}'] = coord.rolling(ws, **roll).std()\n\n if 'nose' in avail_A and 'nose' in avail_B:\n nn = np.sqrt((mouse_pair['A']['nose']['x'] - mouse_pair['B']['nose']['x'])**2 +\n (mouse_pair['A']['nose']['y'] - mouse_pair['B']['nose']['y'])**2)\n for lag in [10, 20, 40]:\n l = _scale(lag, fps)\n X[f'nn_lg{lag}'] = nn.shift(l)\n X[f'nn_ch{lag}'] = nn - nn.shift(l)\n is_cl = (nn < 10.0).astype(float)\n X[f'cl_ps{lag}'] = is_cl.rolling(l, min_periods=1).mean()\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n Avx = mouse_pair['A']['body_center']['x'].diff()\n Avy = mouse_pair['A']['body_center']['y'].diff()\n Bvx = mouse_pair['B']['body_center']['x'].diff()\n Bvy = mouse_pair['B']['body_center']['y'].diff()\n val = (Avx * Bvx + Avy * Bvy) / (np.sqrt(Avx**2 + Avy**2) * np.sqrt(Bvx**2 + Bvy**2) + 1e-6)\n\n for off in [-30, -20, -10, 0, 10, 20, 30]:\n o = _scale_signed(off, fps)\n X[f'va_{off}'] = val.shift(-o)\n\n w = _scale(30, fps)\n X['int_con'] = cd_full.rolling(w, min_periods=1, center=True).std() / \\\n (cd_full.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n return X.astype(np.float32, copy=False)"},{"cell_type":"markdown","id":"d33834e5","metadata":{"papermill":{"duration":0.006121,"end_time":"2025-11-04T09:10:22.884579","exception":false,"start_time":"2025-11-04T09:10:22.878458","status":"completed"},"tags":[]},"source":"# Training, validation and submission"},{"cell_type":"code","execution_count":9,"id":"ba2fe4b2","metadata":{"_kg_hide-input":false,"execution":{"iopub.execute_input":"2025-11-04T09:10:22.90063Z","iopub.status.busy":"2025-11-04T09:10:22.900275Z","iopub.status.idle":"2025-11-04T09:10:22.912602Z","shell.execute_reply":"2025-11-04T09:10:22.911404Z"},"papermill":{"duration":0.022373,"end_time":"2025-11-04T09:10:22.91427","exception":false,"start_time":"2025-11-04T09:10:22.891897","status":"completed"},"tags":[]},"outputs":[],"source":"def robustify(submission, dataset, traintest, traintest_directory=None):\n # Default tracking path based on train/test split\n if traintest_directory is None:\n if traintest == \"train\":\n traintest_directory = CFG.TRAIN_TRACKING_PATH\n else:\n traintest_directory = CFG.TEST_TRACKING_PATH\n\n old_submission = submission.copy()\n submission = submission[submission.start_frame < submission.stop_frame]\n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped frames with start >= stop\")\n \n group_list = []\n for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n group = group.sort_values('start_frame')\n mask = np.ones(len(group), dtype=bool)\n last_stop_frame = 0\n for i, (_, row) in enumerate(group.iterrows()):\n if row['start_frame'] < last_stop_frame:\n mask[i] = False\n else:\n last_stop_frame = row['stop_frame']\n group_list.append(group[mask])\n \n submission = pd.concat(group_list)\n \n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped duplicate frames\")\n \n s_list = []\n for idx, row in dataset.iterrows():\n lab_id = row['lab_id']\n if lab_id.startswith('MABe22'):\n continue\n \n video_id = row['video_id']\n if (submission.video_id == video_id).any():\n continue\n \n if type(row.behaviors_labeled) != str:\n continue\n\n print(f\"Video {video_id} has no predictions.\")\n \n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n vid = pd.read_parquet(path)\n \n vid_behaviors = json.loads(row['behaviors_labeled'])\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n start_frame = vid.video_frame.min()\n stop_frame = vid.video_frame.max() + 1\n \n for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n for i, (_, action_row) in enumerate(actions.iterrows()):\n batch_start = start_frame + i * batch_length\n batch_stop = min(batch_start + batch_length, stop_frame)\n s_list.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n\n if len(s_list) > 0:\n submission = pd.concat([\n submission,\n pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n ])\n print(\"ERROR: Filled empty videos\")\n\n submission = submission.reset_index(drop=True)\n \n return submission"},{"cell_type":"code","execution_count":10,"id":"61376816","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.929528Z","iopub.status.busy":"2025-11-04T09:10:22.929192Z","iopub.status.idle":"2025-11-04T09:10:22.938427Z","shell.execute_reply":"2025-11-04T09:10:22.937402Z"},"papermill":{"duration":0.018167,"end_time":"2025-11-04T09:10:22.940045","exception":false,"start_time":"2025-11-04T09:10:22.921878","status":"completed"},"tags":[]},"outputs":[],"source":"def predict_multiclass(pred, meta, thresholds):\n ama = np.argmax(pred.values, axis=1)\n max_proba = pred.max(axis=1).values\n\n threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n action_thresholds = threshold_array[ama]\n\n ama = np.where(max_proba >= action_thresholds, ama, -1)\n ama = pd.Series(ama, index=meta.video_frame)\n \n changes_mask = (ama != ama.shift(1)).values\n ama_changes = ama[changes_mask]\n meta_changes = meta[changes_mask]\n \n mask = ama_changes.values >= 0\n mask[-1] = False\n \n submission_part = pd.DataFrame({\n 'video_id': meta_changes['video_id'][mask].values,\n 'agent_id': meta_changes['agent_id'][mask].values,\n 'target_id': meta_changes['target_id'][mask].values,\n 'action': pred.columns[ama_changes[mask].values],\n 'start_frame': ama_changes.index[mask],\n 'stop_frame': ama_changes.index[1:][mask[:-1]]\n })\n \n stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n for i in range(len(submission_part)):\n video_id = submission_part.video_id.iloc[i]\n agent_id = submission_part.agent_id.iloc[i]\n target_id = submission_part.target_id.iloc[i]\n if stop_video_id[i] != video_id or stop_agent_id[i] != agent_id or stop_target_id[i] != target_id:\n new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n\n return submission_part"},{"cell_type":"code","execution_count":11,"id":"fca4fcde","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.955261Z","iopub.status.busy":"2025-11-04T09:10:22.954898Z","iopub.status.idle":"2025-11-04T09:10:22.961379Z","shell.execute_reply":"2025-11-04T09:10:22.96047Z"},"papermill":{"duration":0.016284,"end_time":"2025-11-04T09:10:22.963015","exception":false,"start_time":"2025-11-04T09:10:22.946731","status":"completed"},"tags":[]},"outputs":[],"source":"def tune_threshold(oof_action, y_action):\n def objective(trial):\n threshold = trial.suggest_float(\"threshold\", 0, 1, step=0.01)\n return f1_score(y_action, (oof_action >= threshold), zero_division=0)\n\n study = optuna.create_study(direction=\"maximize\")\n study.optimize(objective, n_trials=1000, n_jobs=-1)\n return study.best_params[\"threshold\"]"},{"cell_type":"code","execution_count":12,"id":"257b4c92","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:22.97991Z","iopub.status.busy":"2025-11-04T09:10:22.979566Z","iopub.status.idle":"2025-11-04T09:10:22.990498Z","shell.execute_reply":"2025-11-04T09:10:22.989285Z"},"papermill":{"duration":0.021937,"end_time":"2025-11-04T09:10:22.991894","exception":false,"start_time":"2025-11-04T09:10:22.969957","status":"completed"},"tags":[]},"outputs":[],"source":"def cross_validate_classifier(\n X: pd.DataFrame,\n label: pd.DataFrame,\n meta: pd.DataFrame,\n body_parts_tracked_str: str,\n section: int,\n):\n \"\"\"Train per-action binary classifiers with CV and tuned thresholds.\"\"\"\n oof = pd.DataFrame(index=meta.video_frame)\n\n f1_list = []\n submission_list = []\n thresholds = {}\n\n for action in label.columns:\n action_mask = ~label[action].isna().values\n y_action = label[action][action_mask].values.astype(int)\n X_action = X[action_mask]\n \n # -----------------------------\n # Decide grouping for CV\n # -----------------------------\n if CFG.group_by_lab:\n # 優先用 lab_id 分組；若 lab 數量不足，再退回用 video_id\n groups_action_lab = meta.get(\"lab_id\")\n if groups_action_lab is not None:\n groups_action_lab = groups_action_lab[action_mask]\n if (\n groups_action_lab is not None\n and len(np.unique(groups_action_lab)) >= CFG.n_splits\n ):\n groups_action = groups_action_lab\n else:\n groups_action = meta.video_id[action_mask]\n else:\n # 一律用 video_id 分組（原 baseline 行為）\n groups_action = meta.video_id[action_mask]\n\n # Need enough groups for K-fold CV\n if len(np.unique(groups_action)) < CFG.n_splits:\n continue\n\n if not (y_action == 0).all():\n try:\n with warnings.catch_warnings():\n warnings.filterwarnings(\n \"ignore\", category=RuntimeWarning\n )\n\n # -----------------------------\n # Class imbalance handling\n # -----------------------------\n pos = int(y_action.sum())\n neg = int(len(y_action) - pos)\n if CFG.use_scale_pos_weight:\n # 針對該 action 的正負例比例，調整 scale_pos_weight\n if pos == 0:\n scale_pos_weight = 1.0\n else:\n scale_pos_weight = neg / max(pos, 1)\n else:\n # 不使用額外的類別權重，回到 XGB 預設\n scale_pos_weight = 1.0\n\n # Clone base model and set per-action scale_pos_weight\n estimator = clone(CFG.model)\n estimator.set_params(scale_pos_weight=scale_pos_weight)\n\n # -----------------------------\n # Model save path\n # -----------------------------\n save_path = os.path.join(\n CFG.model_path,\n CFG.model_name,\n str(section),\n action,\n )\n os.makedirs(save_path, exist_ok=True)\n\n trainer = Trainer(\n estimator=estimator,\n cv=CFG.cv,\n cv_args={\"groups\": groups_action},\n metric=f1_score,\n task=\"binary\",\n verbose=False,\n save=True,\n save_path=save_path,\n )\n\n trainer.fit(X_action, y_action)\n oof_action = trainer.oof_preds\n\n threshold = tune_threshold(oof_action, y_action)\n thresholds[action] = threshold\n\n f1 = f1_score(\n y_action,\n (oof_action >= threshold),\n zero_division=0,\n )\n f1_list.append(\n (body_parts_tracked_str, action, f1)\n )\n\n joblib.dump(\n oof_action,\n os.path.join(save_path, \"oof_pred_probs.pkl\"),\n )\n joblib.dump(\n threshold,\n os.path.join(save_path, \"threshold.pkl\"),\n )\n\n print(\n f\"\tF1: {f1:.4f} ({threshold:.2f}) \"\n f\"Section: {section} Action: {action} \"\n f\"[pos={pos}, neg={neg}, spw={scale_pos_weight:.1f}]\"\n )\n\n del trainer\n gc.collect()\n\n except Exception as e:\n print(\n f\"\tException in action {action}: {e}. \"\n \"Falling back to zeros.\"\n )\n oof_action = np.zeros(len(y_action))\n print(\n f\"\tF1: 0.0000 (0.00) Section: {section} Action: {action}\"\n )\n\n else:\n # No positive samples for this action\n oof_action = np.zeros(len(y_action))\n print(f\"\tF1: 0.0000 (0.00) Section: {section} Action: {action}\")\n\n # Place oof predictions back into full index\n oof_column = np.zeros(len(label))\n oof_column[action_mask] = oof_action\n oof[action] = oof_column\n\n del oof_action, action_mask, X_action, y_action, groups_action\n gc.collect()\n\n # Convert per-frame oof probabilities to segments\n submission_part = predict_multiclass(oof, meta, thresholds)\n submission_list.append(submission_part)\n\n return submission_list, f1_list, thresholds"},{"cell_type":"code","execution_count":13,"id":"eab90882","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:10:23.009059Z","iopub.status.busy":"2025-11-04T09:10:23.008104Z","iopub.status.idle":"2025-11-04T09:10:23.017881Z","shell.execute_reply":"2025-11-04T09:10:23.016508Z"},"papermill":{"duration":0.019651,"end_time":"2025-11-04T09:10:23.019416","exception":false,"start_time":"2025-11-04T09:10:22.999765","status":"completed"},"tags":[]},"outputs":[],"source":"def submit(body_parts_tracked_str, switch_tr, section, thresholds):\n \"\"\"Run inference on test set for a given body-parts-tracked configuration.\"\"\"\n body_parts_tracked = json.loads(body_parts_tracked_str)\n if len(body_parts_tracked) > 5:\n body_parts_tracked = [\n b for b in body_parts_tracked if b not in drop_body_parts\n ]\n\n test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n\n generator = generate_mouse_data(\n test_subset,\n \"test\",\n generate_single=(switch_tr == \"single\"),\n generate_pair=(switch_tr == \"pair\"),\n )\n\n fps_lookup = (\n test_subset[[\"video_id\", \"frames_per_second\"]]\n .drop_duplicates(\"video_id\")\n .set_index(\"video_id\")[\"frames_per_second\"]\n .to_dict()\n )\n\n submission_list = []\n for switch_te, data_te, meta_te, actions_te in generator:\n assert switch_te == switch_tr\n try:\n fps_i = _fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n\n # -------------------- pose-based features --------------------\n if switch_te == \"single\":\n X_te = transform_single(data_te, body_parts_tracked, fps_i)\n else:\n X_te = transform_pair(data_te, body_parts_tracked, fps_i)\n del data_te\n gc.collect()\n\n # -------------------- metadata features ----------------------\n meta_te = meta_te.reset_index(drop=True)\n\n if CFG.use_metadata:\n if switch_te == \"single\":\n # Single: one agent per frame (target is 'self')\n mouse_keys = list(\n zip(meta_te[\"video_id\"], meta_te[\"agent_id\"])\n )\n mouse_feat = mouse_meta.reindex(mouse_keys).reset_index(\n drop=True\n )\n arena_feat = arena_meta.reindex(\n meta_te[\"video_id\"]\n ).reset_index(drop=True)\n meta_features = pd.concat(\n [mouse_feat, arena_feat], axis=1\n )\n else:\n # Pair: agent + target mice + arena\n agent_keys = list(\n zip(meta_te[\"video_id\"], meta_te[\"agent_id\"])\n )\n target_keys = list(\n zip(meta_te[\"video_id\"], meta_te[\"target_id\"])\n )\n video_ids = meta_te[\"video_id\"]\n\n agent_feat = mouse_meta.reindex(agent_keys).reset_index(\n drop=True\n )\n target_feat = mouse_meta.reindex(target_keys).reset_index(\n drop=True\n )\n arena_feat = arena_meta.reindex(\n video_ids\n ).reset_index(drop=True)\n\n # Same-strain indicator\n strain_agent = agent_feat.filter(like=\"strain_\").values\n strain_target = target_feat.filter(like=\"strain_\").values\n same_strain = (\n (strain_agent * strain_target).sum(axis=1) > 0\n ).astype(int)\n same_strain = pd.Series(\n same_strain, name=\"same_strain\"\n )\n\n agent_feat = agent_feat.add_prefix(\"agent_\")\n target_feat = target_feat.add_prefix(\"target_\")\n arena_feat = arena_feat.add_prefix(\"arena_\")\n\n meta_features = pd.concat(\n [agent_feat, target_feat, arena_feat, same_strain],\n axis=1,\n )\n\n meta_features.index = X_te.index\n X_te = pd.concat([X_te, meta_features], axis=1).astype(\n np.float32\n )\n\n # -------------------- load models & predict -------------------\n pred = pd.DataFrame(index=meta_te.video_frame)\n for action in actions_te:\n files = glob.glob(\n os.path.join(\n str(section),\n action,\n \"*_trainer_*.pkl\",\n )\n )\n if len(files) == 1:\n trainer = joblib.load(files[0])\n pred[action] = trainer.predict(X_te)\n\n del trainer\n gc.collect()\n\n del X_te\n gc.collect()\n\n if pred.shape[1] != 0:\n submission_part = predict_multiclass(\n pred, meta_te, thresholds\n )\n submission_list.append(submission_part)\n\n except KeyError:\n # Some metadata or tracking might be missing for this video\n del data_te\n gc.collect()\n\n return submission_list\n"},{"cell_type":"code","execution_count":14,"id":"aea99b74","metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["1/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', 'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright', 'lateral_left', 'lateral_right', 'neck', 'nose', 'tail_base', 'tail_midpoint', 'tail_tip']\n"]},{"name":"stdout","output_type":"stream","text":["\tF1: 0.6807 (0.29) Section: 1 Action: rear [pos=46328, neg=498531, spw=1.0]\n","\tF1: 0.3823 (0.13) Section: 1 Action: approach [pos=5365, neg=1088197, spw=1.0]\n","\tF1: 0.3557 (0.06) Section: 1 Action: attack [pos=2563, neg=1144917, spw=1.0]\n","\tF1: 0.4193 (0.14) Section: 1 Action: avoid [pos=12187, neg=1512719, spw=1.0]\n","\tF1: 0.4724 (0.07) Section: 1 Action: chase [pos=4218, neg=1251098, spw=1.0]\n","\tF1: 0.5188 (0.08) Section: 1 Action: chaseattack [pos=1194, neg=1146286, spw=1.0]\n","\tF1: 0.0927 (0.01) Section: 1 Action: submit [pos=682, neg=1092880, spw=1.0]\n","\n","2/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'hip_left', 'hip_right', 'lateral_left', 'lateral_right', 'nose', 'spine_1', 'spine_2', 'tail_base', 'tail_middle_1', 'tail_middle_2', 'tail_tip']\n","\tF1: 0.6204 (0.19) Section: 2 Action: huddle [pos=24447, neg=139924, spw=1.0]\n","\tF1: 0.0000 (0.00) Section: 2 Action: rear\n","\tF1: 0.0000 (0.00) Section: 2 Action: selfgroom\n","\tF1: 0.6001 (0.23) Section: 2 Action: reciprocalsniff [pos=18218, neg=310524, spw=1.0]\n","\tF1: 0.0000 (0.00) Section: 2 Action: sniff\n","\tF1: 0.4658 (0.32) Section: 2 Action: sniffgenital [pos=40972, neg=572744, spw=1.0]\n","\tF1: 0.0000 (0.00) Section: 2 Action: intromit\n","\tF1: 0.0000 (0.00) Section: 2 Action: mount\n","\n","3/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'lateral_left', 'lateral_right', 'neck', 'nose', 'tail_base', 'tail_midpoint', 'tail_tip']\n","\tF1: 0.5230 (0.39) Section: 3 Action: rear [pos=40419, neg=1901466, spw=1.0]\n","\tF1: 0.4185 (0.11) Section: 3 Action: approach [pos=3029, neg=5440441, spw=1.0]\n","\tF1: 0.2355 (0.16) Section: 3 Action: attack [pos=30919, neg=5576111, spw=1.0]\n","\tF1: 0.2610 (0.11) Section: 3 Action: avoid [pos=14017, neg=5511233, spw=1.0]\n","\tF1: 0.0380 (0.05) Section: 3 Action: chase [pos=12349, neg=5512901, spw=1.0]\n","\tF1: 0.2397 (0.03) Section: 3 Action: chaseattack [pos=4343, neg=5520907, spw=1.0]\n","\tF1: 0.1304 (0.26) Section: 3 Action: submit [pos=7882, neg=5435588, spw=1.0]\n","\n","4/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'lateral_left', 'lateral_right', 'nose', 'tail_base', 'tail_tip']\n","\tF1: 0.3634 (0.16) Section: 4 Action: attack [pos=12071, neg=2198106, spw=1.0]\n","\tF1: 0.6291 (0.12) Section: 4 Action: dominance [pos=38483, neg=285516, spw=1.0]\n","\tF1: 0.4795 (0.09) Section: 4 Action: sniff [pos=25068, neg=298931, spw=1.0]\n","\tF1: 0.1236 (0.02) Section: 4 Action: chase [pos=1973, neg=1884205, spw=1.0]\n","\tF1: 0.1719 (0.04) Section: 4 Action: escape [pos=2255, neg=1883923, spw=1.0]\n","\tF1: 0.7024 (0.34) Section: 4 Action: follow [pos=34635, neg=1203705, spw=1.0]\n","\n","5/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'lateral_left', 'lateral_right', 'nose', 'tail_base']\n","\tF1: 0.7273 (0.36) Section: 5 Action: attack [pos=59169, neg=523383, spw=1.0]\n","\tF1: 0.8809 (0.42) Section: 5 Action: sniff [pos=70041, neg=232978, spw=1.0]\n","\tF1: 0.5565 (0.31) Section: 5 Action: defend [pos=10275, neg=187745, spw=1.0]\n","\tF1: 0.7093 (0.23) Section: 5 Action: escape [pos=5905, neg=174115, spw=1.0]\n","\tF1: 0.8358 (0.20) Section: 5 Action: mount [pos=9416, neg=152584, spw=1.0]\n","\n","6/9Processing videos with: ['body_center', 'ear_left', 'ear_right', 'nose', 'tail_base']\n","\tF1: 0.1005 (0.02) Section: 6 Action: biteobject [pos=2359, neg=706137, spw=1.0]\n","\tF1: 0.4639 (0.09) Section: 6 Action: climb [pos=52642, neg=655854, spw=1.0]\n","\tF1: 0.5265 (0.21) Section: 6 Action: dig [pos=41354, neg=667142, spw=1.0]\n","\tF1: 0.1915 (0.06) Section: 6 Action: exploreobject [pos=3783, neg=704713, spw=1.0]\n","\tF1: 0.3510 (0.11) Section: 6 Action: rear [pos=41489, neg=667007, spw=1.0]\n","\tF1: 0.3990 (0.16) Section: 6 Action: selfgroom [pos=35218, neg=673278, spw=1.0]\n","\tF1: 0.5263 (0.19) Section: 6 Action: shepherd [pos=29652, neg=9474762, spw=1.0]\n","\tF1: 0.5004 (0.20) Section: 6 Action: approach [pos=25115, neg=683381, spw=1.0]\n","\tF1: 0.5271 (0.08) Section: 6 Action: attack [pos=12702, neg=695794, spw=1.0]\n","\tF1: 0.6050 (0.12) Section: 6 Action: chase [pos=6970, neg=701526, spw=1.0]\n","\tF1: 0.4104 (0.12) Section: 6 Action: defend [pos=16927, neg=691569, spw=1.0]\n","\tF1: 0.6278 (0.32) Section: 6 Action: escape [pos=27329, neg=681167, spw=1.0]\n","\tF1: 0.0564 (0.01) Section: 6 Action: flinch [pos=1861, neg=706635, spw=1.0]\n","\tF1: 0.3714 (0.06) Section: 6 Action: follow [pos=5742, neg=702754, spw=1.0]\n","\tF1: 0.4820 (0.20) Section: 6 Action: sniff [pos=31819, neg=676677, spw=1.0]\n","\tF1: 0.6198 (0.23) Section: 6 Action: sniffface [pos=21195, neg=687301, spw=1.0]\n","\tF1: 0.1920 (0.02) Section: 6 Action: sniffgenital [pos=2314, neg=706182, spw=1.0]\n","\tF1: 0.2486 (0.01) Section: 6 Action: tussle [pos=5178, neg=703318, spw=1.0]\n","\n","7/9Processing videos with: ['ear_left', 'ear_right', 'head', 'tail_base']\n","\tF1: 0.5574 (0.24) Section: 7 Action: rear [pos=51959, neg=847175, spw=1.0]\n","\tF1: 0.4727 (0.13) Section: 7 Action: rest [pos=87789, neg=443070, spw=1.0]\n","\tF1: 0.3399 (0.20) Section: 7 Action: selfgroom [pos=23216, neg=875918, spw=1.0]\n","\tF1: 0.3528 (0.24) Section: 7 Action: climb [pos=8702, neg=287194, spw=1.0]\n","\tF1: 0.3380 (0.17) Section: 7 Action: dig [pos=31703, neg=740073, spw=1.0]\n","\tF1: 0.1214 (0.12) Section: 7 Action: run [pos=1808, neg=412114, spw=1.0]\n","\tF1: 0.0000 (0.00) Section: 7 Action: intromit\n","\tF1: 0.0000 (0.00) Section: 7 Action: mount\n","\tF1: 0.6643 (0.32) Section: 7 Action: sniff [pos=84890, neg=814244, spw=1.0]\n","\tF1: 0.5205 (0.09) Section: 7 Action: sniffgenital [pos=15444, neg=746733, spw=1.0]\n","\tF1: 0.4495 (0.29) Section: 7 Action: approach [pos=19871, neg=809411, spw=1.0]\n","\tF1: 0.0813 (0.08) Section: 7 Action: defend [pos=981, neg=180946, spw=1.0]\n","\tF1: 0.2702 (0.20) Section: 7 Action: escape [pos=6489, neg=708808, spw=1.0]\n","\tF1: 0.2569 (0.02) Section: 7 Action: attemptmount [pos=1162, neg=97026, spw=1.0]\n","\n","8/9Processing videos with: ['ear_left', 'ear_right', 'hip_left', 'hip_right', 'neck', 'nose', 'tail_base']\n","\tF1: 0.2533 (0.13) Section: 8 Action: rear [pos=36018, neg=2316069, spw=1.0]\n","\tF1: 0.2386 (0.07) Section: 8 Action: selfgroom [pos=24681, neg=2122639, spw=1.0]\n","\tF1: 0.6708 (0.20) Section: 8 Action: genitalgroom [pos=6320, neg=96125, spw=1.0]\n","\tF1: 0.0380 (0.08) Section: 8 Action: dig [pos=6840, neg=558999, spw=1.0]\n","\tF1: 0.4988 (0.25) Section: 8 Action: approach [pos=27759, neg=1009052, spw=1.0]\n","\tF1: 0.6707 (0.32) Section: 8 Action: attack [pos=334896, neg=4379932, spw=1.0]\n","\tF1: 0.4973 (0.25) Section: 8 Action: disengage [pos=12300, neg=197006, spw=1.0]\n","\tF1: 0.6840 (0.36) Section: 8 Action: mount [pos=278217, neg=5875871, spw=1.0]\n","\tF1: 0.6975 (0.36) Section: 8 Action: sniff [pos=1948652, neg=8870000, spw=1.0]\n","\tF1: 0.6568 (0.27) Section: 8 Action: sniffgenital [pos=653061, neg=7113259, spw=1.0]\n","\tF1: 0.4724 (0.16) Section: 8 Action: dominancemount [pos=17966, neg=1216099, spw=1.0]\n","\tF1: 0.5792 (0.27) Section: 8 Action: sniffbody [pos=106792, neg=782162, spw=1.0]\n","\tF1: 0.6779 (0.30) Section: 8 Action: sniffface [pos=53143, neg=693462, spw=1.0]\n","\tF1: 0.1903 (0.08) Section: 8 Action: attemptmount [pos=4972, neg=694734, spw=1.0]\n","\tF1: 0.8451 (0.32) Section: 8 Action: intromit [pos=348476, neg=2557612, spw=1.0]\n","\tF1: 0.1009 (0.04) Section: 8 Action: chase [pos=2012, neg=615853, spw=1.0]\n","\tF1: 0.5682 (0.37) Section: 8 Action: escape [pos=8691, neg=707148, spw=1.0]\n","\tF1: 0.7589 (0.39) Section: 8 Action: reciprocalsniff [pos=23956, neg=276044, spw=1.0]\n","\tF1: 0.1338 (0.09) Section: 8 Action: allogroom [pos=7624, neg=864891, spw=1.0]\n","\tF1: 0.2082 (0.01) Section: 8 Action: ejaculate [pos=1608, neg=160864, spw=1.0]\n","\tF1: 0.1809 (0.03) Section: 8 Action: dominancegroom [pos=4777, neg=561062, spw=1.0]\n","\n","9/9Processing videos with: ['ear_left', 'ear_right', 'nose', 'tail_base', 'tail_tip']\n","\tF1: 0.3793 (0.04) Section: 9 Action: freeze [pos=31765, neg=298012, spw=1.0]\n","\tF1: 0.3393 (0.25) Section: 9 Action: rear [pos=19071, neg=236696, spw=1.0]\n","\tF1: 0.1628 (0.01) Section: 9 Action: approach [pos=8099, neg=320467, spw=1.0]\n","\tF1: 0.6933 (0.38) Section: 9 Action: attack [pos=72816, neg=515833, spw=1.0]\n","\tF1: 0.5459 (0.24) Section: 9 Action: defend [pos=63208, neg=786734, spw=1.0]\n","\tF1: 0.6783 (0.39) Section: 9 Action: escape [pos=41706, neg=474049, spw=1.0]\n","\tF1: 0.5711 (0.06) Section: 9 Action: sniff [pos=27577, neg=269619, spw=1.0]\n","\n"]}],"source":"# Containers for CV results and thresholds\nf1_list = []\noof_submission_list = []\n\nthresholds = {\n \"single\": {},\n \"pair\": {},\n}\n\nfor section in range(1, len(body_parts_tracked_list)):\n body_parts_tracked_str = body_parts_tracked_list[section]\n try:\n body_parts_tracked = json.loads(body_parts_tracked_str)\n print(f\"{section}/{len(body_parts_tracked_list)-1}\"\n f\"Processing videos with: {body_parts_tracked}\")\n\n if len(body_parts_tracked) > 5:\n body_parts_tracked = [\n b for b in body_parts_tracked if b not in drop_body_parts\n ]\n\n train_subset = train[train.body_parts_tracked == body_parts_tracked_str]\n\n _fps_lookup = (\n train_subset[[\"video_id\", \"frames_per_second\"]]\n .drop_duplicates(\"video_id\")\n .set_index(\"video_id\")[\"frames_per_second\"]\n .to_dict()\n )\n\n # ----------------------- single-mouse models -----------------------\n single_mouse_list = []\n single_mouse_meta_list = []\n single_mouse_label_list = []\n\n generator_single = generate_mouse_data(\n train_subset,\n \"train\",\n generate_single=True,\n generate_pair=False,\n )\n\n for switch_tr, data_tr, meta_tr, label_tr in generator_single:\n # switch_tr is always \"single\" here\n single_mouse_list.append(data_tr)\n single_mouse_meta_list.append(meta_tr)\n single_mouse_label_list.append(label_tr)\n del data_tr, meta_tr, label_tr\n gc.collect()\n\n if len(single_mouse_list) > 0:\n single_feats_parts = []\n for data_i, meta_i in zip(\n single_mouse_list, single_mouse_meta_list\n ):\n fps_i = _fps_from_meta(meta_i, _fps_lookup, default_fps=30.0)\n X_i = transform_single(\n data_i, body_parts_tracked, fps_i\n ).astype(np.float32)\n single_feats_parts.append(X_i)\n del X_i, fps_i\n gc.collect()\n\n X_tr = pd.concat(single_feats_parts, axis=0, ignore_index=True)\n single_mouse_label = pd.concat(\n single_mouse_label_list, axis=0, ignore_index=True\n )\n single_mouse_meta = pd.concat(\n single_mouse_meta_list, axis=0, ignore_index=True\n )\n\n del (\n single_feats_parts,\n single_mouse_list,\n single_mouse_label_list,\n single_mouse_meta_list,\n )\n gc.collect()\n\n # ----- attach metadata features (agent + arena) -----\n if CFG.use_metadata:\n single_mouse_meta = single_mouse_meta.reset_index(drop=True)\n mouse_keys = list(\n zip(\n single_mouse_meta[\"video_id\"],\n single_mouse_meta[\"agent_id\"],\n )\n )\n mfeat = mouse_meta.reindex(mouse_keys).reset_index(drop=True)\n afeat = arena_meta.reindex(\n single_mouse_meta[\"video_id\"]\n ).reset_index(drop=True)\n meta_features = pd.concat([mfeat, afeat], axis=1)\n meta_features.index = X_tr.index\n X_tr = pd.concat([X_tr, meta_features], axis=1).astype(\n np.float32\n )\n\n # CV & threshold tuning\n temp_submission_list, temp_f1_list, temp_thresholds = (\n cross_validate_classifier(\n X_tr,\n single_mouse_label,\n single_mouse_meta,\n body_parts_tracked_str,\n section,\n )\n )\n\n\n if f\"{section}\" not in thresholds[\"single\"]:\n thresholds[\"single\"][f\"{section}\"] = {}\n for k, v in temp_thresholds.items():\n thresholds[\"single\"][f\"{section}\"][k] = v\n\n f1_list.extend(temp_f1_list)\n oof_submission_list.extend(temp_submission_list)\n\n del (\n temp_submission_list,\n temp_f1_list,\n temp_thresholds,\n X_tr,\n )\n gc.collect()\n\n # ----------------------- pair-wise models --------------------------\n mouse_pair_list = []\n mouse_pair_meta_list = []\n mouse_pair_label_list = []\n\n generator_pair = generate_mouse_data(\n train_subset,\n \"train\",\n generate_single=False,\n generate_pair=True,\n )\n\n for switch_tr, data_tr, meta_tr, label_tr in generator_pair:\n # switch_tr is always \"pair\" here\n mouse_pair_list.append(data_tr)\n mouse_pair_meta_list.append(meta_tr)\n mouse_pair_label_list.append(label_tr)\n del data_tr, meta_tr, label_tr\n gc.collect()\n\n if len(mouse_pair_list) > 0:\n pair_feats_parts = []\n for data_i, meta_i in zip(\n mouse_pair_list, mouse_pair_meta_list\n ):\n fps_i = _fps_from_meta(meta_i, _fps_lookup, default_fps=30.0)\n X_i = transform_pair(\n data_i, body_parts_tracked, fps_i\n ).astype(np.float32)\n pair_feats_parts.append(X_i)\n del X_i, fps_i\n gc.collect()\n\n X_tr = pd.concat(pair_feats_parts, axis=0, ignore_index=True)\n mouse_pair_label = pd.concat(\n mouse_pair_label_list, axis=0, ignore_index=True\n )\n mouse_pair_meta = pd.concat(\n mouse_pair_meta_list, axis=0, ignore_index=True\n )\n\n del (\n pair_feats_parts,\n mouse_pair_list,\n mouse_pair_label_list,\n mouse_pair_meta_list,\n )\n gc.collect()\n\n # ----- attach metadata features (agent + target + arena) -----\n if CFG.use_metadata:\n mouse_pair_meta = mouse_pair_meta.reset_index(drop=True)\n agent_keys = list(\n zip(\n mouse_pair_meta[\"video_id\"],\n mouse_pair_meta[\"agent_id\"],\n )\n )\n target_keys = list(\n zip(\n mouse_pair_meta[\"video_id\"],\n mouse_pair_meta[\"target_id\"],\n )\n )\n video_ids = mouse_pair_meta[\"video_id\"]\n\n agent_feat = mouse_meta.reindex(agent_keys).reset_index(\n drop=True\n )\n target_feat = mouse_meta.reindex(target_keys).reset_index(\n drop=True\n )\n arena_feat = arena_meta.reindex(video_ids).reset_index(\n drop=True\n )\n\n # Same-strain indicator\n strain_agent = agent_feat.filter(like=\"strain_\").values\n strain_target = target_feat.filter(like=\"strain_\").values\n same_strain = (\n (strain_agent * strain_target).sum(axis=1) > 0\n ).astype(int)\n same_strain = pd.Series(\n same_strain, name=\"same_strain\"\n )\n\n agent_feat = agent_feat.add_prefix(\"agent_\")\n target_feat = target_feat.add_prefix(\"target_\")\n arena_feat = arena_feat.add_prefix(\"arena_\")\n\n meta_features = pd.concat(\n [agent_feat, target_feat, arena_feat, same_strain],\n axis=1,\n )\n meta_features.index = X_tr.index\n X_tr = pd.concat([X_tr, meta_features], axis=1).astype(\n np.float32\n )\n\n # CV & threshold tuning\n temp_submission_list, temp_f1_list, temp_thresholds = (\n cross_validate_classifier(\n X_tr,\n mouse_pair_label,\n mouse_pair_meta,\n body_parts_tracked_str,\n section,\n )\n )\n\n\n if f\"{section}\" not in thresholds[\"pair\"]:\n thresholds[\"pair\"][f\"{section}\"] = {}\n for k, v in temp_thresholds.items():\n thresholds[\"pair\"][f\"{section}\"][k] = v\n\n f1_list.extend(temp_f1_list)\n oof_submission_list.extend(temp_submission_list)\n\n del (\n temp_submission_list,\n temp_f1_list,\n temp_thresholds,\n X_tr,\n )\n gc.collect()\n\n except Exception as e:\n print(f\"\tException in section {section}: {e}\")\n print()"},{"cell_type":"code","execution_count":15,"id":"0be4d791","metadata":{},"outputs":[{"data":{"application/vnd.jupyter.widget-view+json":{"model_id":"a278dfbd321041bb8b968c90761e4887","version_major":2,"version_minor":0},"text/plain":[" 0%| | 0/863 [00:00 0:\n submission_oof = pd.concat(oof_submission_list, ignore_index=True)\n submission_oof_robust = robustify(submission_oof, train, \"train\")\n print(\n f\"Competition metric (CV): \"\n f\"{score(solution, submission_oof_robust, ''):.4f}\"\n )\n\n f1_df = pd.DataFrame(\n f1_list,\n columns=[\"body_parts_tracked_str\", \"action\", \"binary F1 score\"],\n )\n print(\n f\"Mean binary F1 over actions: \"\n f\"{f1_df['binary F1 score'].mean():.4f}\"\n )\n\n # Save thresholds and CV scores\n thresholds_path = os.path.join(CFG.model_path, CFG.model_name, \"thresholds.pkl\")\n scores_path = os.path.join(CFG.model_path, CFG.model_name, \"scores.pkl\")\n os.makedirs(os.path.dirname(thresholds_path), exist_ok=True)\n joblib.dump(thresholds, thresholds_path)\n joblib.dump(f1_df, scores_path)\nelse:\n print(\"No OOF submissions were generated.\")"},{"cell_type":"code","execution_count":16,"id":"d4994b85","metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["===== Experiment configuration =====\n","group_by_lab : False\n","use_metadata : True\n","use_scale_pos_weight: False\n","Competition metric (CV): 0.4766\n","Mean binary F1 over actions: 0.4331\n"]}],"source":"print(\"===== Experiment configuration =====\")\nprint(f\"group_by_lab : {CFG.group_by_lab}\")\nprint(f\"use_metadata : {CFG.use_metadata}\")\nprint(f\"use_scale_pos_weight: {CFG.use_scale_pos_weight}\")\n\nif \"submission_oof_robust\" in globals():\n metric = score(solution, submission_oof_robust, \"\")\n mean_f1 = f1_df[\"binary F1 score\"].mean()\n print(f\"Competition metric (CV): {metric:.4f}\")\n print(f\"Mean binary F1 over actions: {mean_f1:.4f}\")\nelse:\n print(\"No OOF predictions available in this run.\")"},{"cell_type":"markdown","id":"f64a25b7","metadata":{"papermill":{"duration":0.006773,"end_time":"2025-11-04T09:34:24.557959","exception":false,"start_time":"2025-11-04T09:34:24.551186","status":"completed"},"tags":[]},"source":"# Submission"},{"cell_type":"code","execution_count":17,"id":"51686586","metadata":{"execution":{"iopub.execute_input":"2025-11-04T09:34:24.572693Z","iopub.status.busy":"2025-11-04T09:34:24.572398Z","iopub.status.idle":"2025-11-04T09:34:24.676244Z","shell.execute_reply":"2025-11-04T09:34:24.675302Z"},"papermill":{"duration":0.113312,"end_time":"2025-11-04T09:34:24.678004","exception":false,"start_time":"2025-11-04T09:34:24.564692","status":"completed"},"tags":[]},"outputs":[{"data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
video_idagent_idtarget_idactionstart_framestop_frame
row_id
0438887472mouse1selfrear278500
\n","
"],"text/plain":[" video_id agent_id target_id action start_frame stop_frame\n","row_id \n","0 438887472 mouse1 self rear 278 500"]},"execution_count":17,"metadata":{},"output_type":"execute_result"}],"source":"submission_test_list = []\n\nfor section in range(1, len(body_parts_tracked_list)):\n body_parts_tracked_str = body_parts_tracked_list[section]\n\n # Single-mouse actions\n if f\"{section}\" in thresholds[\"single\"]:\n sub_single = submit(\n body_parts_tracked_str,\n \"single\",\n section,\n thresholds[\"single\"][f\"{section}\"],\n )\n submission_test_list.extend(sub_single)\n\n # Pairwise actions\n if f\"{section}\" in thresholds[\"pair\"]:\n sub_pair = submit(\n body_parts_tracked_str,\n \"pair\",\n section,\n thresholds[\"pair\"][f\"{section}\"],\n )\n submission_test_list.extend(sub_pair)\n\nif len(submission_test_list) > 0:\n submission = pd.concat(submission_test_list, ignore_index=True)\nelse:\n # Fallback: extremely small dummy submission to satisfy format\n submission = pd.DataFrame(\n dict(\n video_id=438887472,\n agent_id=\"mouse1\",\n target_id=\"self\",\n action=\"rear\",\n start_frame=278,\n stop_frame=500,\n ),\n index=[0],\n )\n\nsubmission_robust = robustify(submission, test, \"test\")\nsubmission_robust.index.name = \"row_id\"\nsubmission_robust.to_csv(\"submission.csv\")\n\nsubmission_robust.head()"}],"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"databundleVersionId":13874099,"sourceId":59156,"sourceType":"competition"},{"datasetId":8619229,"sourceId":13607639,"sourceType":"datasetVersion"}],"dockerImageVersionId":31089,"isGpuEnabled":false,"isInternetEnabled":false,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"aiml","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.18"},"papermill":{"default_parameters":{},"duration":1455.03842,"end_time":"2025-11-04T09:34:26.113884","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-11-04T09:10:11.075464","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00962fe9010e42b7a5b123af50d2a026":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b098fa23a3b4031bf88b32b20044978":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"108c3e5db1154f47a5aca68386ed4f24":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"38c36db3be3843b19da108d4cd1f9421":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e9806911d1d489d963f76955d8468c4","IPY_MODEL_e6ad60bc760e4b51885b15335c048a65","IPY_MODEL_6f4723180bf14101b60deeac9fd59e50"],"layout":"IPY_MODEL_00962fe9010e42b7a5b123af50d2a026","tabbable":null,"tooltip":null}},"3e9806911d1d489d963f76955d8468c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_aeb2544f9ad9413b8b71b5f5643b1019","placeholder":"​","style":"IPY_MODEL_ec012e9ac53d477daee0988a35117f0b","tabbable":null,"tooltip":null,"value":"100%"}},"3edd91e1eafb4f6bbf55275ea60b6c97":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f4723180bf14101b60deeac9fd59e50":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e8f95c8096294ce6b6fb2b65c08af536","placeholder":"​","style":"IPY_MODEL_0b098fa23a3b4031bf88b32b20044978","tabbable":null,"tooltip":null,"value":" 863/863 [00:08<00:00, 96.46it/s]"}},"aeb2544f9ad9413b8b71b5f5643b1019":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e6ad60bc760e4b51885b15335c048a65":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_3edd91e1eafb4f6bbf55275ea60b6c97","max":863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_108c3e5db1154f47a5aca68386ed4f24","tabbable":null,"tooltip":null,"value":863}},"e8f95c8096294ce6b6fb2b65c08af536":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec012e9ac53d477daee0988a35117f0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":5}