{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13561817,"sourceType":"datasetVersion","datasetId":8614395},{"sourceId":13792775,"sourceType":"datasetVersion","datasetId":8659206},{"sourceId":13850274,"sourceType":"datasetVersion","datasetId":8817771},{"sourceId":13966208,"sourceType":"datasetVersion","datasetId":8894871},{"sourceId":14042587,"sourceType":"datasetVersion","datasetId":8933070},{"sourceId":14085657,"sourceType":"datasetVersion","datasetId":8968091},{"sourceId":14109288,"sourceType":"datasetVersion","datasetId":8932968},{"sourceId":14109557,"sourceType":"datasetVersion","datasetId":8987385,"isSourceIdPinned":true},{"sourceId":14138921,"sourceType":"datasetVersion","datasetId":8987181},{"sourceId":14139218,"sourceType":"datasetVersion","datasetId":9010389},{"sourceId":14155465,"sourceType":"datasetVersion","datasetId":9016304,"isSourceIdPinned":true},{"sourceId":14164558,"sourceType":"datasetVersion","datasetId":9028557},{"sourceId":273367188,"sourceType":"kernelVersion"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":1387.894195,"end_time":"2025-10-30T12:37:58.021268","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-30T12:14:50.127073","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00962fe9010e42b7a5b123af50d2a026":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b098fa23a3b4031bf88b32b20044978":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"108c3e5db1154f47a5aca68386ed4f24":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"38c36db3be3843b19da108d4cd1f9421":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e9806911d1d489d963f76955d8468c4","IPY_MODEL_e6ad60bc760e4b51885b15335c048a65","IPY_MODEL_6f4723180bf14101b60deeac9fd59e50"],"layout":"IPY_MODEL_00962fe9010e42b7a5b123af50d2a026","tabbable":null,"tooltip":null}},"3e9806911d1d489d963f76955d8468c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_aeb2544f9ad9413b8b71b5f5643b1019","placeholder":"​","style":"IPY_MODEL_ec012e9ac53d477daee0988a35117f0b","tabbable":null,"tooltip":null,"value":"100%"}},"3edd91e1eafb4f6bbf55275ea60b6c97":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f4723180bf14101b60deeac9fd59e50":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e8f95c8096294ce6b6fb2b65c08af536","placeholder":"​","style":"IPY_MODEL_0b098fa23a3b4031bf88b32b20044978","tabbable":null,"tooltip":null,"value":" 863/863 [00:08<00:00, 96.46it/s]"}},"aeb2544f9ad9413b8b71b5f5643b1019":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e6ad60bc760e4b51885b15335c048a65":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_3edd91e1eafb4f6bbf55275ea60b6c97","max":863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_108c3e5db1154f47a5aca68386ed4f24","tabbable":null,"tooltip":null,"value":863}},"e8f95c8096294ce6b6fb2b65c08af536":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec012e9ac53d477daee0988a35117f0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**References**\n- [MABe Nearest Neighbors: The Original ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-nearest-neighbors-the-original)\n- [MABe EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-eda-which-makes-sense)\n- [MABe Validated baseline without machine learning](https://www.kaggle.com/code/ambrosm/mabe-validated-baseline-without-machine-learning)\n- [Squeeze GBT](https://www.kaggle.com/code/cody11null/squeeze-gbt)","metadata":{"_uuid":"75a5c58b-609d-4679-be72-e4389b3e99e2","_cell_guid":"e690cc98-c2ad-4a09-81b9-154395f724ba","trusted":true,"collapsed":false,"papermill":{"duration":0.00879,"end_time":"2025-10-30T12:14:55.751776","exception":false,"start_time":"2025-10-30T12:14:55.742986","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"_uuid":"bcff7504-0c91-40da-a307-3fbc2c1855d3","_cell_guid":"21ff30a2-3d73-490b-ac30-dbd835a43bec","trusted":true,"collapsed":false,"papermill":{"duration":0.006291,"end_time":"2025-10-30T12:14:55.764738","exception":false,"start_time":"2025-10-30T12:14:55.758447","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## install module","metadata":{"_uuid":"a834291b-d2f1-43ac-8471-48e8f3b30509","_cell_guid":"7f5ef21f-7a81-4263-8df0-39b96b410e5b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"!pip install koolbox","metadata":{"_uuid":"eb5fc82c-fbbb-45f9-a0a9-1469619330a8","_cell_guid":"5694dbe3-7d29-4408-87e6-e79d8841445e","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:30.740403Z","iopub.execute_input":"2025-12-14T16:49:30.740814Z","iopub.status.idle":"2025-12-14T16:49:40.036677Z","shell.execute_reply.started":"2025-12-14T16:49:30.740789Z","shell.execute_reply":"2025-12-14T16:49:40.035622Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## import Module","metadata":{"_uuid":"f527f870-fbd5-466e-8657-4b3688f3b1b9","_cell_guid":"d01554bb-2c70-4506-b98e-8f16aed6704b","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"import gc\nimport json\nimport itertools\nimport glob\nimport joblib\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport warnings\nfrom pathlib import Path\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass\nfrom typing import (\n Optional, \n Union, Literal, \n Generator, \n Tuple, \n List, \n Dict, \n TYPE_CHECKING, \n NamedTuple, \n Any\n)\n\nimport optuna\nfrom koolbox import Trainer\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.base import clone\nfrom xgboost import XGBClassifier\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"31267010-e164-48dd-9fe3-6e1b6bc03084","_cell_guid":"968ba099-ea7f-4e03-990f-f58c68fed285","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:40.038718Z","iopub.execute_input":"2025-12-14T16:49:40.039428Z","iopub.status.idle":"2025-12-14T16:49:41.037277Z","shell.execute_reply.started":"2025-12-14T16:49:40.039396Z","shell.execute_reply":"2025-12-14T16:49:41.035989Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## set const","metadata":{"_uuid":"7f3d6dcd-1b45-4ba4-b571-bbd74feb613a","_cell_guid":"cafe54bc-73e4-4ecb-ae3c-5376550a0945","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"DROP_BODY_PARTS = [\n 'headpiece_bottombackleft', 'headpiece_bottombackright', \n 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n 'headpiece_topbackleft', 'headpiece_topbackright', \n 'headpiece_topfrontleft', 'headpiece_topfrontright', \n 'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint', \"neck\", \"tail_tip\"\n]\n\nUNUSE_LAB = [\n \"CalMS21_supplemental\",\n \"CalMS21_task1\",\n \"CalMS21_task2\",\n \"MABe22_keypoints\",\n \"MABe22_movies\"\n]\n\nDROP_FEATURE_COL = {\n \n}","metadata":{"_uuid":"1b659d88-30ad-466d-94b2-124246804ece","_cell_guid":"f6c31d82-ffc5-45c3-b28e-e6d6b612da1b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:41.037955Z","iopub.execute_input":"2025-12-14T16:49:41.038331Z","iopub.status.idle":"2025-12-14T16:49:41.044032Z","shell.execute_reply.started":"2025-12-14T16:49:41.038311Z","shell.execute_reply":"2025-12-14T16:49:41.042938Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## set config","metadata":{"_uuid":"701dc2b0-9453-4a38-9735-d97c0f56ebf3","_cell_guid":"77817707-cbe0-4090-a239-99d5391d4be4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class CFG:\n root_dir = \"/kaggle/input/MABe-mouse-behavior-detection\"\n\n model_path = \"/kaggle/input/mabe-xgb-v7\"\n model_name = \"xgboost\"\n \n # mode = \"validate\"\n mode = \"submit\"\n \n n_splits = 3\n cv = StratifiedGroupKFold(n_splits)\n\n model = XGBClassifier(\n verbosity=0, \n # tree_method= \"gpu_hist\", # GPUを使用\n # device= \"cuda\", # CUDA対応GPUを指定\n random_state=42,\n n_estimators=250, \n learning_rate=0.08, \n max_depth=6,\n min_child_weight=5, \n subsample=0.8, \n colsample_bytree=0.8\n )\n\n do_modeling = False\n do_tuning = True","metadata":{"_uuid":"60f0aad6-75d3-47a2-8623-2f4b1982e540","_cell_guid":"0b63b6f9-a60e-46ef-bb6a-b16050486eea","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:44.634742Z","iopub.execute_input":"2025-12-14T16:49:44.635056Z","iopub.status.idle":"2025-12-14T16:49:44.641161Z","shell.execute_reply.started":"2025-12-14T16:49:44.635035Z","shell.execute_reply":"2025-12-14T16:49:44.640103Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helpers","metadata":{"_uuid":"97ca1a8a-3f49-46c3-b3c1-8ec080d362dc","_cell_guid":"b99dda93-f80a-4e96-b75b-05c3997449b5","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"## Data Loader","metadata":{"_uuid":"78b9c8fe-cc29-40de-b49e-87194659a90c","_cell_guid":"218c7d69-9a65-4f14-9784-3b8d3bf44343","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class DataLoader:\n \"\"\"\n DataLoader class for MABe mouse behavior detection dataset.\n Provides unified interface for loading all data files.\n \n Args:\n root_dir: Root directory of the dataset\n \"\"\"\n \n def __init__(self, root_dir: Union[str, Path]):\n \"\"\"\n Initialize DataLoader with root directory.\n \n Args:\n root_dir: Path to the root directory containing train.csv, test.csv, etc.\n \"\"\"\n self.root_dir = Path(root_dir)\n self.train_csv_path = self.root_dir / \"train.csv\"\n self.test_csv_path = self.root_dir / \"test.csv\"\n self.train_annotation_dir = self.root_dir / \"train_annotation\"\n self.train_tracking_dir = self.root_dir / \"train_tracking\"\n self.test_tracking_dir = self.root_dir / \"test_tracking\"\n \n # Cache for train and test DataFrames\n self._train_df: Optional[pd.DataFrame] = None\n self._test_df: Optional[pd.DataFrame] = None\n\n @property\n def lab_list(self):\n self.load_train_csv(add_n_mice=False, exclude_unuse_lab=False)\n return list(np.unique(self._train_df.lab_id))\n \n def load_train_csv(\n self, \n force_reload: bool = False,\n add_n_mice: bool = True,\n exclude_unuse_lab: bool = True\n ) -> pd.DataFrame:\n \"\"\"\n Load train.csv file with optional preprocessing.\n \n Args:\n force_reload: If True, reload from file even if cached\n add_n_mice: If True, add 'n_mice' column (default: True)\n exclude_mabe22: If True, exclude rows with lab_id starting with 'MABe22_' (default: False)\n \n Returns:\n DataFrame containing train data\n \n Note:\n - n_mice is calculated as: 4 - number of missing mouse strains\n - When exclude_mabe22=True, filters out videos from MABe22 labs\n \"\"\"\n if self._train_df is None or force_reload:\n self._train_df = pd.read_csv(self.train_csv_path)\n \n # Create a copy to avoid modifying the cached version\n train_df = self._train_df.copy()\n \n # Add n_mice column\n if add_n_mice:\n train_df['n_mice'] = 4 - train_df[[\n 'mouse1_strain', 'mouse2_strain', 'mouse3_strain', 'mouse4_strain'\n ]].isna().sum(axis=1)\n \n # Exclude MABe22, etc data if requested\n if exclude_unuse_lab:\n train_df = train_df[~train_df.lab_id.isin(UNUSE_LAB)]\n train_df = train_df[~((train_df[\"lab_id\"] == \"AdaptableSnail\") & (train_df[\"frames_per_second\"] == 25))]\n \n return train_df\n \n def load_test_csv(self, force_reload: bool = False) -> pd.DataFrame:\n \"\"\"\n Load test.csv file.\n \n Args:\n force_reload: If True, reload from file even if cached\n \n Returns:\n DataFrame containing test data\n \"\"\"\n if self._test_df is None or force_reload:\n self._test_df = pd.read_csv(self.test_csv_path)\n\n test_df = self._test_df.copy()\n test_df = test_df[~((test_df[\"lab_id\"] == \"AdaptableSnail\") & (test_df[\"frames_per_second\"] == 25))]\n return self._test_df\n \n def load_train_annotation(self, lab_id: str, video_id: str) -> pd.DataFrame:\n \"\"\"\n Load annotation file for a specific training video.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n DataFrame containing annotation data\n \n Raises:\n FileNotFoundError: If annotation file does not exist\n \"\"\"\n annotation_path = self.train_annotation_dir / lab_id / f\"{video_id}.parquet\"\n return pd.read_parquet(annotation_path)\n \n def load_train_tracking(self, lab_id: str, video_id: str) -> pd.DataFrame:\n \"\"\"\n Load tracking file for a specific training video.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n DataFrame containing tracking data\n \n Raises:\n FileNotFoundError: If tracking file does not exist\n \"\"\"\n tracking_path = self.train_tracking_dir / lab_id / f\"{video_id}.parquet\"\n return pd.read_parquet(tracking_path)\n \n def load_test_tracking(self, lab_id: str, video_id: str) -> pd.DataFrame:\n \"\"\"\n Load tracking file for a specific test video.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n DataFrame containing tracking data\n \n Raises:\n FileNotFoundError: If tracking file does not exist\n \"\"\"\n tracking_path = self.test_tracking_dir / lab_id / f\"{video_id}.parquet\"\n return pd.read_parquet(tracking_path)\n \n def annotation_exists(self, lab_id: str, video_id: str) -> bool:\n \"\"\"\n Check if annotation file exists.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n True if annotation file exists, False otherwise\n \"\"\"\n annotation_path = self.train_annotation_dir / lab_id / f\"{video_id}.parquet\"\n return annotation_path.exists()\n \n def train_tracking_exists(self, lab_id: str, video_id: str) -> bool:\n \"\"\"\n Check if train tracking file exists.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n True if tracking file exists, False otherwise\n \"\"\"\n tracking_path = self.train_tracking_dir / lab_id / f\"{video_id}.parquet\"\n return tracking_path.exists()\n \n def test_tracking_exists(self, lab_id: str, video_id: str) -> bool:\n \"\"\"\n Check if test tracking file exists.\n \n Args:\n lab_id: Laboratory ID\n video_id: Video ID\n \n Returns:\n True if tracking file exists, False otherwise\n \"\"\"\n tracking_path = self.test_tracking_dir / lab_id / f\"{video_id}.parquet\"\n return tracking_path.exists()\n \n def create_solution_df(self, exclude_unuse_lab: bool = True) -> pd.DataFrame:\n \"\"\"\n Create solution DataFrame by loading and processing annotation data from train dataset.\n \n This method loads train.csv, then loads annotation files for each video,\n adds metadata columns, and formats agent_id and target_id.\n \n Args:\n exclude_mabe22: If True, exclude videos with lab_id starting with 'MABe22' (default: True)\n \n Returns:\n DataFrame containing concatenated annotation data with additional metadata\n \n Note:\n - Uses load_train_csv() internally to get the train dataset\n - Skips videos without annotation files\n - Converts agent_id to format 'mouseX'\n - Converts target_id to format 'mouseX' or 'self'\n \"\"\"\n # Load train dataset\n train_df = self.load_train_csv(add_n_mice=False, exclude_unuse_lab=exclude_unuse_lab)\n \n solution = []\n for _, row in tqdm(train_df.iterrows(), total=len(train_df)):\n \n lab_id = row['lab_id']\n video_id = row['video_id']\n \n # Use the new interface to load annotation\n try:\n annot = self.load_train_annotation(lab_id, video_id)\n except FileNotFoundError:\n continue\n \n # Add metadata columns\n annot['lab_id'] = lab_id\n annot['video_id'] = video_id\n annot['behaviors_labeled'] = row['behaviors_labeled']\n \n # Format target_id and agent_id\n annot['target_id'] = np.where(\n annot.target_id != annot.agent_id, \n annot['target_id'].apply(lambda s: f\"mouse{s}\"), \n 'self'\n )\n annot['agent_id'] = annot['agent_id'].apply(lambda s: f\"mouse{s}\")\n solution.append(annot)\n \n solution = pd.concat(solution)\n \n return solution","metadata":{"_uuid":"bcb42244-2891-4e27-9034-58426cba487d","_cell_guid":"fafd8e19-b7ad-4d13-981a-5ff0da85146b","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:45.651435Z","iopub.execute_input":"2025-12-14T16:49:45.652051Z","iopub.status.idle":"2025-12-14T16:49:45.672386Z","shell.execute_reply.started":"2025-12-14T16:49:45.652027Z","shell.execute_reply":"2025-12-14T16:49:45.671388Z"},"_kg_hide-output":true,"_kg_hide-input":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Preprocessor","metadata":{"_uuid":"c043d1f3-7653-414c-a9f9-adf427459f98","_cell_guid":"91f8e3f8-75e9-44be-bfbb-8c0a34303ff0","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class UnpredictionException(Exception):\n \"\"\"バリデーションエラー\"\"\"\n def __init__(self, message):\n self.message = message\n super().__init__(self.message)\n\n@dataclass\nclass BehaviorData:\n \"\"\"\n Base data class for mouse behavior data.\n \n Attributes:\n features: DataFrame containing tracking features\n metadata: DataFrame containing metadata (video_id, agent_id, target_id, video_frame)\n fps: Frames per second for the video\n feature_store: FeatureStore instance for generating features (optional)\n labels: DataFrame with binary labels (for train mode, None for test mode)\n actions: Array of action names (for test mode, None for train mode)\n \"\"\"\n features: pd.DataFrame\n metadata: pd.DataFrame\n fps: float\n feature_store: Optional[Union['SingleFeatureStore', 'PairFeatureStore']] = None\n labels: Optional[pd.DataFrame] = None\n actions: Optional[np.ndarray] = None\n \n @property\n def behavior_type(self) -> Literal['single', 'pair']:\n \"\"\"Return the behavior type.\"\"\"\n raise NotImplementedError\n \n def generate_features(self) -> pd.DataFrame:\n \"\"\"\n Generate features using the injected feature_store.\n \n Returns:\n DataFrame containing generated features\n \n Raises:\n ValueError: If feature_store is not set\n \"\"\"\n if self.feature_store is None:\n raise ValueError(\"feature_store is not set. Cannot generate features.\")\n return self.feature_store.generate(self.features)\n \n @staticmethod\n def _create_labels(\n annot: pd.DataFrame,\n agent_id: int,\n target_id: int,\n actions: np.ndarray,\n video_frames: pd.Index\n ) -> pd.DataFrame:\n \"\"\"\n Create label DataFrame from annotation data.\n \n Args:\n annot: Annotation DataFrame\n agent_id: Agent mouse ID (integer)\n target_id: Target mouse ID (integer)\n actions: Array of action names\n video_frames: Index of video frame numbers\n \n Returns:\n DataFrame with binary labels\n \"\"\"\n label_df = pd.DataFrame(0.0, columns=actions, index=video_frames)\n annot_subset = annot.query(\"(agent_id == @agent_id) & (target_id == @target_id)\")\n \n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n label_df.loc[annot_row['start_frame']:annot_row['stop_frame'], annot_row.action] = 1.0\n \n return label_df\n\ndef add_body_center_if_needed(ppvid: pd.DataFrame) -> pd.DataFrame:\n \"\"\"\n Add body_center to the DataFrame if it doesn't exist.\n \n Priority:\n 1. If body_center already exists, return as is\n 2. If nose and tail_base exist, add body_center as their midpoint\n 3. If head and tail_base exist, add body_center as their midpoint\n \n Args:\n ppvid: DataFrame with MultiIndex columns (bodypart, coordinate)\n where coordinate is 'x' or 'y'\n \n Returns:\n DataFrame with body_center added if it was missing\n \"\"\"\n # Check if body_center already exists\n if 'body_center' in ppvid.columns.get_level_values(0):\n return ppvid\n \n # Try to create body_center from nose and tail_base\n if 'nose' in ppvid.columns.get_level_values(0) and 'tail_base' in ppvid.columns.get_level_values(0):\n ppvid[('body_center', 'x')] = (ppvid[('nose', 'x')] + ppvid[('tail_base', 'x')]) / 2\n ppvid[('body_center', 'y')] = (ppvid[('nose', 'y')] + ppvid[('tail_base', 'y')]) / 2\n return ppvid\n \n # Try to create body_center from head and tail_base\n if 'head' in ppvid.columns.get_level_values(0) and 'tail_base' in ppvid.columns.get_level_values(0):\n ppvid[('body_center', 'x')] = (ppvid[('head', 'x')] + ppvid[('tail_base', 'x')]) / 2\n ppvid[('body_center', 'y')] = (ppvid[('head', 'y')] + ppvid[('tail_base', 'y')]) / 2\n return ppvid\n \n # If none of the conditions are met, return as is\n ppvid[('body_center', 'x')] = np.nan\n ppvid[('body_center', 'y')] = np.nan\n return ppvid\n\n\n@dataclass\nclass SingleBehaviorData(BehaviorData):\n \"\"\"\n Data class for single mouse behavior data.\n \"\"\"\n \n @property\n def behavior_type(self) -> Literal['single', 'pair']:\n return 'single'\n \n @staticmethod\n def extract_agents(vid_behaviors: pd.DataFrame) -> List[str]:\n \"\"\"\n Extract list of single mouse agents from behaviors DataFrame.\n \n Args:\n vid_behaviors: DataFrame with columns ['agent', 'target', 'action']\n \n Returns:\n List of agent IDs (e.g., ['mouse1', 'mouse2'])\n \"\"\"\n vid_behaviors_single = vid_behaviors.query(\"target == 'self'\")\n return list(np.unique(vid_behaviors_single.agent))\n \n @staticmethod\n def _extract_features_and_metadata(\n pvid: pd.DataFrame,\n mouse_id_str: str,\n video_id: str\n ) -> Tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"\n Extract single mouse tracking data and create metadata.\n \n Args:\n pvid: Normalized tracking data in wide format\n mouse_id_str: Mouse identifier string (e.g., 'mouse1')\n video_id: Video identifier\n \n Returns:\n Tuple of (features, metadata)\n \"\"\"\n mouse_id = int(mouse_id_str[-1])\n features = pvid.loc[:, mouse_id]\n features = add_body_center_if_needed(features)\n assert len(features) == len(pvid)\n \n metadata = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': mouse_id_str,\n 'target_id': 'self',\n 'video_frame': features.index\n })\n \n return features, metadata\n \n @classmethod\n def create(\n cls,\n pvid: pd.DataFrame,\n mouse_id_str: str,\n video_id: str,\n vid_behaviors: pd.DataFrame,\n fps: float,\n feature_store: Optional['SingleFeatureStore'] = None,\n annot: Optional[pd.DataFrame] = None\n ) -> 'SingleBehaviorData':\n \"\"\"\n Create SingleBehaviorData from tracking and annotation data.\n \n Args:\n pvid: Normalized tracking data in wide format\n mouse_id_str: Mouse identifier string (e.g., 'mouse1')\n video_id: Video identifier\n vid_behaviors: DataFrame of all behaviors (will be filtered internally)\n fps: Frames per second for the video\n feature_store: SingleFeatureStore instance for feature generation (optional)\n annot: Annotation DataFrame (required for train mode)\n \n Returns:\n SingleBehaviorData instance\n \"\"\"\n # Extract features and metadata\n features, metadata = cls._extract_features_and_metadata(pvid, mouse_id_str, video_id)\n \n # Filter to single mouse behaviors and get actions for this mouse\n vid_behaviors_single = vid_behaviors.query(\"target == 'self'\")\n vid_agent_actions = np.unique(vid_behaviors_single.query(\"agent == @mouse_id_str\").action)\n \n # Create labels or set actions\n if annot is not None:\n mouse_id = int(mouse_id_str[-1])\n labels = cls._create_labels(annot, mouse_id, mouse_id, vid_agent_actions, features.index)\n return cls(\n features=features,\n metadata=metadata,\n fps=fps,\n feature_store=feature_store,\n labels=labels,\n actions=vid_agent_actions\n )\n else:\n return cls(\n features=features,\n metadata=metadata,\n fps=fps,\n feature_store=feature_store,\n labels=None,\n actions=vid_agent_actions\n )\n \n @classmethod\n def create_all(\n cls,\n pvid: pd.DataFrame,\n video_id: str,\n vid_behaviors: pd.DataFrame,\n fps: float,\n feature_store: Optional['SingleFeatureStore'] = None,\n annot: Optional[pd.DataFrame] = None\n ) -> Generator['SingleBehaviorData', None, None]:\n \"\"\"\n Create all SingleBehaviorData instances for a video.\n \n Args:\n pvid: Normalized tracking data in wide format\n video_id: Video identifier\n vid_behaviors: DataFrame of all behaviors\n fps: Frames per second for the video\n feature_store: SingleFeatureStore instance for feature generation (optional)\n annot: Annotation DataFrame (required for train mode)\n \n Yields:\n SingleBehaviorData instances for each single mouse agent\n \"\"\"\n single_agents = cls.extract_agents(vid_behaviors)\n for mouse_id_str in single_agents:\n try:\n yield cls.create(pvid, mouse_id_str, video_id, vid_behaviors, fps, feature_store, annot)\n except KeyError:\n pass\n\n\n@dataclass\nclass PairBehaviorData(BehaviorData):\n \"\"\"\n Data class for mouse pair behavior data.\n \"\"\"\n \n @property\n def behavior_type(self) -> Literal['single', 'pair']:\n return 'pair'\n \n @staticmethod\n def extract_pairs(vid_behaviors: pd.DataFrame) -> List[Tuple[str, str]]:\n \"\"\"\n Extract list of mouse pairs from behaviors DataFrame.\n \n Args:\n vid_behaviors: DataFrame with columns ['agent', 'target', 'action']\n \n Returns:\n List of (agent, target) tuples (e.g., [('mouse1', 'mouse2'), ('mouse2', 'mouse1')])\n \"\"\"\n vid_behaviors_pair = vid_behaviors.query(\"target != 'self'\")\n if len(vid_behaviors_pair) > 0:\n return list(vid_behaviors_pair[['agent', 'target']].drop_duplicates().values)\n return []\n \n @staticmethod\n def _extract_features_and_metadata(\n pvid: pd.DataFrame,\n agent_str: str,\n target_str: str,\n video_id: str\n ) -> Tuple[pd.DataFrame, pd.DataFrame]:\n \"\"\"\n Extract mouse pair tracking data and create metadata.\n \n Args:\n pvid: Normalized tracking data in wide format\n agent_str: Agent mouse identifier string (e.g., 'mouse1')\n target_str: Target mouse identifier string (e.g., 'mouse2')\n video_id: Video identifier\n \n Returns:\n Tuple of (features, metadata)\n \"\"\"\n agent = int(agent_str[-1])\n target = int(target_str[-1])\n agent_features = add_body_center_if_needed(pvid[agent])\n target_features = add_body_center_if_needed(pvid[target])\n features = pd.concat([agent_features, target_features], axis=1, keys=['A', 'B'])\n assert len(features) == len(pvid)\n \n metadata = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': agent_str,\n 'target_id': target_str,\n 'video_frame': features.index\n })\n \n return features, metadata\n \n @classmethod\n def create(\n cls,\n pvid: pd.DataFrame,\n agent_str: str,\n target_str: str,\n video_id: str,\n vid_behaviors: pd.DataFrame,\n fps: float,\n feature_store: Optional['PairFeatureStore'] = None,\n annot: Optional[pd.DataFrame] = None\n ) -> 'PairBehaviorData':\n \"\"\"\n Create PairBehaviorData from tracking and annotation data.\n \n Args:\n pvid: Normalized tracking data in wide format\n agent_str: Agent mouse identifier string (e.g., 'mouse1')\n target_str: Target mouse identifier string (e.g., 'mouse2')\n video_id: Video identifier\n vid_behaviors: DataFrame of all behaviors (will be filtered internally)\n fps: Frames per second for the video\n feature_store: PairFeatureStore instance for feature generation (optional)\n annot: Annotation DataFrame (required for train mode)\n \n Returns:\n PairBehaviorData instance\n \"\"\"\n # Extract features and metadata\n features, metadata = cls._extract_features_and_metadata(pvid, agent_str, target_str, video_id)\n \n # Filter to pair behaviors and get actions for this pair\n vid_behaviors_pair = vid_behaviors.query(\"target != 'self'\")\n vid_agent_actions = np.unique(\n vid_behaviors_pair.query(\"(agent == @agent_str) & (target == @target_str)\").action\n )\n \n # Create labels or set actions\n if annot is not None:\n agent = int(agent_str[-1])\n target = int(target_str[-1])\n labels = cls._create_labels(annot, agent, target, vid_agent_actions, features.index)\n return cls(\n features=features,\n metadata=metadata,\n fps=fps,\n feature_store=feature_store,\n labels=labels,\n actions=vid_agent_actions\n )\n else:\n return cls(\n features=features,\n metadata=metadata,\n fps=fps,\n feature_store=feature_store,\n labels=None,\n actions=vid_agent_actions\n )\n \n @classmethod\n def create_all(\n cls,\n pvid: pd.DataFrame,\n video_id: str,\n vid_behaviors: pd.DataFrame,\n fps: float,\n feature_store: Optional['PairFeatureStore'] = None,\n annot: Optional[pd.DataFrame] = None\n ) -> Generator['PairBehaviorData', None, None]:\n \"\"\"\n Create all PairBehaviorData instances for a video.\n \n Args:\n pvid: Normalized tracking data in wide format\n video_id: Video identifier\n vid_behaviors: DataFrame of all behaviors\n fps: Frames per second for the video\n feature_store: PairFeatureStore instance for feature generation (optional)\n annot: Annotation DataFrame (required for train mode)\n \n Yields:\n PairBehaviorData instances for each mouse pair\n \"\"\"\n pair_combinations = cls.extract_pairs(vid_behaviors)\n for agent_str, target_str in pair_combinations:\n try:\n yield cls.create(pvid, agent_str, target_str, video_id, vid_behaviors, fps, feature_store, annot)\n except KeyError:\n pass\n\nclass Preprocessor:\n \"\"\"\n Preprocessor class for MABe mouse behavior detection dataset.\n Handles data preprocessing and transformation after loading.\n \n Args:\n loader: DataLoader instance for accessing data files\n mode: 'train' or 'test' mode\n dataset: DataFrame containing video metadata (train.csv or test.csv)\n body_parts_tracked: Optional list of body parts to use (if None, will be computed from dataset)\n \"\"\"\n \n def __init__(\n self,\n loader: 'DataLoader',\n mode: Literal['train', 'test'],\n dataset: pd.DataFrame,\n body_parts_tracked: Optional[List[str]] = None\n ):\n \"\"\"\n Initialize Preprocessor with necessary data.\n \n Args:\n loader: DataLoader instance for accessing data files\n mode: 'train' or 'test' - determines which tracking/annotation to load\n dataset: DataFrame containing video metadata (must have 'lab_id', 'video_id',\n 'behaviors_labeled', 'pix_per_cm_approx' columns)\n body_parts_tracked: Optional list of body parts to use. If provided, this will be used\n instead of computing from dataset. Useful for test mode to ensure\n consistency with train mode.\n \"\"\"\n self.loader = loader\n self.mode = mode\n self.dataset = dataset\n self._body_parts_tracked_cache: Optional[List[str]] = body_parts_tracked\n \n @property\n def body_parts_tracked(self) -> List[str]:\n \"\"\"\n Get the filtered list of body parts to use for feature generation.\n \n This property:\n 1. Extracts all unique body_parts_tracked values from the dataset\n 2. Parses JSON strings and collects all body parts\n 3. Removes duplicates\n 4. Filters out DROP_BODY_PARTS if there are more than 5 body parts\n 5. Returns a sorted list\n \n Returns:\n Sorted list of body part names to use\n \"\"\"\n if self._body_parts_tracked_cache is None:\n body_parts = []\n for u in self.dataset.body_parts_tracked.unique():\n body_parts += json.loads(u)\n body_parts = list(set(body_parts))\n if len(body_parts) > 5:\n body_parts = [b for b in body_parts if b not in DROP_BODY_PARTS]\n if \"body_center\" not in body_parts:\n body_parts.append(\"body_center\")\n self._body_parts_tracked_cache = sorted(body_parts)\n return self._body_parts_tracked_cache\n \n @property\n def fps_lookup(self) -> dict:\n \"\"\"\n Get the FPS lookup dictionary mapping video_id to frames_per_second.\n \n Returns:\n Dictionary mapping video_id (str) to frames_per_second (float)\n \"\"\"\n return (\n self.dataset[['video_id', 'frames_per_second']]\n .drop_duplicates('video_id')\n .set_index('video_id')['frames_per_second']\n .to_dict()\n )\n \n def normalize_tracking_data(self, vid: pd.DataFrame, pix_per_cm_approx: float) -> pd.DataFrame:\n \"\"\"\n Normalize tracking data from long format to wide format with coordinate normalization.\n \n This method processes raw tracking data by:\n 1. Dropping unnecessary body parts (if more than 5 unique body parts exist)\n 2. Pivoting from long to wide format\n 3. Reordering column levels\n 4. Normalizing coordinates by pix_per_cm_approx\n \n Args:\n vid: Raw tracking DataFrame in long format with columns \n ['video_frame', 'mouse_id', 'bodypart', 'x', 'y']\n pix_per_cm_approx: Approximate pixels per centimeter for coordinate normalization\n \n Returns:\n Normalized DataFrame in wide format with hierarchical columns\n (mouse_id, bodypart, coordinate) indexed by video_frame\n \n Note:\n - Drops body parts listed in DROP_BODY_PARTS if more than 5 unique body parts\n - Calls garbage collection to free memory after pivot operation\n \"\"\"\n # Drop unnecessary body parts if needed\n if len(np.unique(vid.bodypart)) > 5:\n vid = vid.query(\"~ bodypart.isin(@DROP_BODY_PARTS)\")\n \n # Pivot to get wide format\n pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n \n del vid\n gc.collect()\n \n # Reorder levels and normalize coordinates\n pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n pvid /= pix_per_cm_approx\n \n return pvid\n \n def parse_behaviors_labeled(self, behaviors_labeled_str: str) -> pd.DataFrame:\n \"\"\"\n Parse behaviors_labeled string into a structured DataFrame.\n \n This method processes the behaviors_labeled JSON string by:\n 1. Loading from JSON format\n 2. Removing quotes and sorting/deduplicating\n 3. Splitting by comma into (agent, target, action) tuples\n 4. Converting to DataFrame\n \n Args:\n behaviors_labeled_str: JSON string containing behavior labels\n e.g., '[\"mouse1,self,action1\", \"mouse2,mouse3,action2\"]'\n \n Returns:\n DataFrame with columns ['agent', 'target', 'action']\n \n Example:\n >>> behaviors = '[\"mouse1,self,groom\", \"mouse2,mouse3,chase\"]'\n >>> preprocessor = Preprocessor(loader)\n >>> df = preprocessor.parse_behaviors_labeled(behaviors)\n >>> print(df)\n agent target action\n 0 mouse1 self groom\n 1 mouse2 mouse3 chase\n \"\"\"\n vid_behaviors = json.loads(behaviors_labeled_str)\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action']) \n return vid_behaviors\n \n def generate_mouse_data(\n self,\n generate_single: bool = True,\n generate_pair: bool = True\n ) -> Generator[Union[SingleBehaviorData, PairBehaviorData], None, None]:\n \"\"\"\n Generate mouse data by processing tracking and annotation files.\n \n This method yields processed mouse tracking data along with metadata and labels (for train)\n or action lists (for test). It generates both single-mouse and mouse-pair data.\n \n Args:\n generate_single: If True, generate single mouse data (agent-self behaviors)\n generate_pair: If True, generate mouse pair data (agent-target behaviors)\n \n Yields:\n SingleBehaviorData or PairBehaviorData: Data class containing:\n - features: DataFrame with tracking coordinates\n - metadata: DataFrame with video_id, agent_id, target_id, video_frame\n - fps: Frames per second for the video\n - feature_store: FeatureStore instance for generating features\n - labels: DataFrame with binary labels (train mode only)\n - actions: Array of action names (test mode only)\n - behavior_type: Property that returns 'single' or 'pair'\n \n Note:\n - Uses dataset and mode from initialization\n - Skips videos without valid behaviors_labeled\n - Drops unnecessary body parts if more than 5 unique body parts exist\n - Normalizes coordinates by pix_per_cm_approx\n - Injects FeatureStore into each BehaviorData instance\n \"\"\"\n \n # Get body parts tracked for this dataset\n body_parts = self.body_parts_tracked\n \n for _, row in self.dataset.iterrows():\n # Load tracking data using the appropriate method\n try:\n if self.mode == 'train':\n vid = self.loader.load_train_tracking(row.lab_id, row.video_id)\n else:\n vid = self.loader.load_test_tracking(row.lab_id, row.video_id)\n except FileNotFoundError:\n continue\n\n # Normalize tracking data to wide format\n pvid = self.normalize_tracking_data(vid, row.pix_per_cm_approx)\n\n # Parse behaviors labeled\n try:\n vid_behaviors = self.parse_behaviors_labeled(row.behaviors_labeled)\n except:\n if row.behaviors_labeled is np.nan:\n continue\n raise ValueError\n # Load annotation for training mode\n annot = None\n if self.mode == 'train':\n try:\n annot = self.loader.load_train_annotation(row.lab_id, row.video_id)\n except FileNotFoundError:\n continue\n \n # Get FPS for this video\n fps = self.fps_lookup.get(row.video_id, 30.0)\n \n # Generate single mouse data\n if generate_single:\n single_feature_store = SingleFeatureStore({}, body_parts, fps)\n yield from SingleBehaviorData.create_all(\n pvid=pvid,\n video_id=row.video_id,\n vid_behaviors=vid_behaviors,\n fps=fps,\n feature_store=single_feature_store,\n annot=annot\n )\n\n # Generate mouse pair data\n if generate_pair:\n pair_feature_store = PairFeatureStore({}, body_parts, fps)\n yield from PairBehaviorData.create_all(\n pvid=pvid,\n video_id=row.video_id,\n vid_behaviors=vid_behaviors,\n fps=fps,\n feature_store=pair_feature_store,\n annot=annot\n )","metadata":{"_uuid":"916c32d4-74f0-4876-8076-94772bae2372","_cell_guid":"ca30b56d-aa5a-4098-a173-b3d137ebc885","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:46.093106Z","iopub.execute_input":"2025-12-14T16:49:46.093496Z","iopub.status.idle":"2025-12-14T16:49:46.148207Z","shell.execute_reply.started":"2025-12-14T16:49:46.09347Z","shell.execute_reply":"2025-12-14T16:49:46.147221Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feature","metadata":{"_uuid":"8ac3189a-d773-4759-aae2-8bdd3b4ee92b","_cell_guid":"10c7df3d-79f7-493e-839f-9e57e1edb3e1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"## commons\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n \"\"\"FPSに基づいてフレーム数をスケーリング\"\"\"\n return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n \"\"\"符号を保持してFPSスケーリング\"\"\"\n if n_frames_at_30fps == 0:\n return 0\n s = 1 if n_frames_at_30fps > 0 else -1\n mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n return s * mag\n\ndef calculate_distance(point1, point2=None, squared=False):\n \"\"\"汎用距離計算関数\"\"\"\n if isinstance(point1, tuple):\n x1, y1 = point1\n else:\n x1, y1 = point1['x'], point1['y']\n \n if point2 is not None:\n if isinstance(point2, tuple):\n x2, y2 = point2\n else:\n x2, y2 = point2['x'], point2['y']\n dist_sq = (x1 - x2)**2 + (y1 - y2)**2\n else:\n dist_sq = x1**2 + y1**2\n \n if squared:\n return dist_sq\n else:\n return np.sqrt(dist_sq)\n\ndef calculate_angle_cosine(vector1, vector2):\n \"\"\"2つのベクトル間の角度のコサイン値を計算\"\"\"\n if isinstance(vector1, tuple):\n x1, y1 = vector1\n else:\n x1, y1 = vector1['x'], vector1['y']\n \n if isinstance(vector2, tuple):\n x2, y2 = vector2\n else:\n x2, y2 = vector2['x'], vector2['y']\n \n dot_product = x1 * x2 + y1 * y2\n magnitude1 = calculate_distance((x1, y1))\n magnitude2 = calculate_distance((x2, y2))\n \n cos_angle = dot_product / (magnitude1 * magnitude2 + 1e-6)\n return cos_angle\n\ndef calculate_point_speed(point, lag, squared=False):\n \"\"\"身体部位の移動速度を計算\"\"\"\n shifted = point.shift(lag)\n return calculate_distance(point, shifted, squared=squared)\n\ndef calculate_point_acceleration(point, lag, squared=False):\n \"\"\"身体部位の加速度を計算\"\"\"\n acc = point.diff(lag).diff(lag)\n return calculate_distance(acc, squared=squared)\n\ndef calculate_cross_point_speed(point1, point2, lag, squared=False):\n \"\"\"現在の部位1と過去の部位2間の距離を計算\"\"\"\n shifted_point2 = point2.shift(lag)\n return calculate_distance(point1, shifted_point2, squared=squared)\n\ndef calculate_distance_velocity(distance_series, lag):\n \"\"\"距離の速度（時間変化率）を計算\"\"\"\n return distance_series.diff(lag)\n\ndef calculate_distance_acceleration(distance_series, lag):\n \"\"\"距離の加速度（速度の変化率）を計算\"\"\"\n return distance_series.diff(lag).diff(lag)\n\ndef calculate_coefficient_of_variation(series, window, fps, min_periods=1, center=True):\n \"\"\"変動係数（coefficient of variation: std/mean）を計算\"\"\"\n ws = _scale(window, fps)\n roll_mean = series.rolling(ws, min_periods=min_periods, center=center).mean()\n roll_std = series.rolling(ws, min_periods=min_periods, center=center).std()\n return roll_std / (roll_mean + 1e-6)","metadata":{"_uuid":"e767c198-2d7a-49d3-beb0-25204954424b","_cell_guid":"3985eabd-6698-4d9b-b7b4-1f65eb6f2acb","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:46.291329Z","iopub.execute_input":"2025-12-14T16:49:46.29166Z","iopub.status.idle":"2025-12-14T16:49:46.306985Z","shell.execute_reply.started":"2025-12-14T16:49:46.29164Z","shell.execute_reply":"2025-12-14T16:49:46.305856Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================\n# 基底Generatorクラス\n# ====================\n\nclass BaseGenerator(ABC):\n \"\"\"特徴量生成の基底クラス\"\"\"\n \n def __init__(self, feature_config: Dict, available_body_parts: pd.Index, fps: float):\n \"\"\"\n Parameters\n ----------\n feature_config : Dict\n 特徴量設定（使用するパラメータなど）\n available_body_parts : pd.Index\n 利用可能な身体部位\n fps : float\n フレームレート\n \"\"\"\n self.feature_config = feature_config\n self.available_body_parts = available_body_parts\n self.fps = fps\n \n @property\n def body_parts_tracked(self) -> List[str]:\n \"\"\"追跡する身体部位のリストを取得\"\"\"\n return self.feature_config.get('body_parts_tracked', [])\n \n @property\n def lags(self) -> List[int]:\n \"\"\"ラグのリストを取得\"\"\"\n return self.feature_config.get('lags', [10, 20, 40])\n \n @property\n def left_right_pairs(self) -> List[Tuple[str, str, str]]:\n \"\"\"\n _leftと_rightの接尾語を持つ身体部位のペアを取得\n \n Returns\n -------\n List[Tuple[str, str, str]]\n [(left_part, right_part, base_name), ...]のリスト\n 例: [('ear_left', 'ear_right', 'ear'), ('hand_left', 'hand_right', 'hand')]\n \"\"\"\n pairs = []\n for part in self.body_parts_tracked:\n if part.endswith('_left'):\n base_name = part[:-5] # '_left'を除去\n right_part = f'{base_name}_right'\n if right_part in self.available_body_parts:\n pairs.append((part, right_part, base_name))\n return pairs\n \n @property\n def inter_mouse_parts(self) -> List[str]:\n \"\"\"\n マウス間距離計算に使用する身体部位のリストを取得\n \n Returns\n -------\n List[str]\n 身体部位のリスト\n 例: ['body_center', 'ear_left', 'ear_right']\n \"\"\"\n return self.feature_config.get('inter_mouse_parts', [\n 'body_center', 'ear_left', 'ear_right', \"tail_base\"\n ])\n \n def validate(self) -> bool:\n \"\"\"\n 特徴量生成が可能かどうかを検証\n \n Returns\n -------\n bool\n 生成可能な場合True、スキップすべき場合False\n \n Notes\n -----\n デフォルトではTrueを返す。\n 各具体クラスで必要な身体部位の存在確認などを実装する。\n \"\"\"\n return True\n \n @abstractmethod\n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n \"\"\"\n 特徴量を生成\n \n Parameters\n ----------\n X : pd.DataFrame\n 既存の特徴量DataFrame\n data : pd.DataFrame\n 入力データ\n \n Returns\n -------\n pd.DataFrame\n 特徴量が追加されたDataFrame\n \"\"\"\n pass\n \n def _get_lag(self, lag_frames: int = 10) -> int:\n \"\"\"FPSでスケーリングされたlag値を取得\"\"\"\n return _scale(lag_frames, self.fps)\n \n def _check_parts_available(self, required_parts: List[str], *additional_sources: pd.Index) -> bool:\n \"\"\"\n 必要な身体部位が全て利用可能かチェック\n \n Parameters\n ----------\n required_parts : List[str]\n 必要な身体部位のリスト\n *additional_sources : pd.Index\n 追加で確認する身体部位のインデックス（オプション）\n 指定された場合、self.available_body_partsと全ての追加ソースに\n 部位が存在する必要がある\n \n Returns\n -------\n bool\n 全て利用可能な場合True\n \n Examples\n --------\n >>> # 単一ソースでのチェック\n >>> self._check_parts_available(['nose', 'tail_base'])\n \n >>> # 複数ソースでのチェック（マウスAとマウスBの両方に存在するか）\n >>> self._check_parts_available(['body_center'], avail_A, avail_B)\n \"\"\"\n # まず基本のavailable_body_partsでチェック\n if not all(part in self.available_body_parts for part in required_parts):\n return False\n \n # 追加ソースが指定されている場合、それらもチェック\n for source in additional_sources:\n if not all(part in source for part in required_parts):\n return False\n \n return True\n \n def _generate_intra_mouse_distances(\n self,\n mouse_data: pd.DataFrame,\n available_parts: pd.Index,\n prefix: str = ''\n ) -> pd.DataFrame:\n \"\"\"\n 単一マウス内の全body_part間の距離を計算\n \n Parameters\n ----------\n mouse_data : pd.DataFrame\n マウスのデータ（multi_index: [body_part, x_or_y]）\n available_parts : pd.Index\n 利用可能なbody_part\n prefix : str, optional\n カラム名のプレフィックス（例: 'A_', 'B_'）、デフォルトは空文字\n \n Returns\n -------\n pd.DataFrame\n 距離カラムを含むDataFrame\n \"\"\"\n distances = {}\n for p1, p2 in itertools.combinations(self.body_parts_tracked, 2):\n if p1 in available_parts and p2 in available_parts:\n col_name = f\"{prefix}{p1}+{p2}\"\n distances[col_name] = calculate_distance(mouse_data[p1], mouse_data[p2], squared=True)\n \n # 全カラムでreindex（存在しないものはNaN）\n all_columns = [f\"{prefix}{p1}+{p2}\" for p1, p2 in itertools.combinations(self.body_parts_tracked, 2)]\n result = pd.DataFrame(distances)\n result = result.reindex(columns=all_columns, copy=False)\n \n return result\n\nclass DistanceGenerator(BaseGenerator):\n \"\"\"全point間の距離を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n distances = self._generate_intra_mouse_distances(data, self.available_body_parts, prefix='')\n return pd.concat([X, distances], axis=1)\n\nclass SpeedGenerator(BaseGenerator):\n \"\"\"全身体部位の速度を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n lag = self._get_lag(10)\n \n for part in self.body_parts_tracked:\n if self._check_parts_available([part]):\n X[f'sp_{part}'] = calculate_point_speed(data[part], lag, squared=True)\n else:\n X[f'sp_{part}'] = np.nan\n return X\n\nclass AccelerationGenerator(BaseGenerator):\n \"\"\"全身体部位の加速度を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n lag = self._get_lag(10)\n \n for part in self.body_parts_tracked:\n if self._check_parts_available([part]):\n X[f'acc_{part}'] = calculate_point_acceleration(data[part], lag, squared=True)\n else:\n X[f'acc_{part}'] = np.nan\n \n return X\n\nclass CrossSpeedGenerator(BaseGenerator):\n \"\"\"異なる部位間のクロス速度を生成\"\"\"\n\n @property\n def cross_speed_patterns(self) -> List[Tuple[str, str]]:\n \"\"\"\n クロススピード計算のための身体部位ペアを取得\n \n Returns\n -------\n List[Tuple[str, str]]\n [(part1, part2), ...]のリスト\n 例: [('ear_left', 'tail_base'), ('ear_right', 'tail_base')]\n \"\"\"\n return self.feature_config.get('cross_speed_patterns', [\n ('ear_left', 'tail_base'),\n ('ear_right', 'tail_base'),\n ])\n \n def validate(self) -> bool:\n \"\"\"設定されたパターンのいずれかが利用可能であればTrue\"\"\"\n # 少なくとも1つのパターンが利用可能であればTrue\n for part1, part2 in self.cross_speed_patterns:\n if self._check_parts_available([part1, part2]):\n return True\n return False\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n lag = self._get_lag(10) \n for part1, part2 in self.cross_speed_patterns:\n if not self._check_parts_available([part1, part2]):\n continue\n\n X[f'sp_{part1}_{part2}'] = calculate_cross_point_speed(\n data[part1],\n data[part2],\n lag,\n squared=True\n )\n return X\n\nclass DistanceLagGenerator(BaseGenerator):\n \"\"\"距離特徴量に対してラグと差分を追加\"\"\"\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n distance_columns = [\n f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(self.body_parts_tracked, 2)\n if f\"{p1}+{p2}\" in X.columns\n ]\n \n for col in distance_columns:\n for lag in self.lags:\n l = _scale(lag, self.fps)\n X[f'{col}_lg{lag}'] = X[col].shift(l)\n X[f'{col}_df{lag}'] = X[col] - X[col].shift(l)\n \n return X\n\nclass LeftRightPairGenerator(BaseGenerator):\n \"\"\"_leftと_rightの接尾語を持つpoint間の特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"少なくとも1つのleft-rightペアが必要\"\"\"\n return len(self.left_right_pairs) > 0\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n # 各ペアに対して特徴量を計算\n for left_part, right_part, base_name in self.left_right_pairs:\n lr_dist = calculate_distance(data[left_part], data[right_part])\n \n # 複数オフセットでの差分特徴量\n for off in [-60, -30, 30, 60]:\n o = _scale_signed(off, self.fps)\n X[f'{base_name}_d{off}'] = lr_dist.diff(o)\n \n # 変動係数\n X[f'{base_name}_con'] = calculate_coefficient_of_variation(lr_dist, window=30, fps=self.fps)\n \n return X\n\nclass BodyAngleGenerator(BaseGenerator):\n \"\"\"身体の角度特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"nose, body_center, tail_baseが必要\"\"\"\n return self._check_parts_available(['nose', 'body_center', 'tail_base'])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n v1 = data['nose'] - data['body_center']\n v2 = data['tail_base'] - data['body_center']\n X['body_ang'] = calculate_angle_cosine(v1, v2)\n return X\n\nclass ElongationGenerator(BaseGenerator):\n \"\"\"伸縮率特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"nose, tail_base, ear_left, ear_rightが必要\"\"\"\n return self._check_parts_available([\n 'nose', 'tail_base', \n 'ear_left', 'ear_right'\n ])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n # nose+tail_baseとear_left+ear_rightの距離カラムが生成済みであることを確認\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n return X\n\nclass BodyCenterMovementGenerator(BaseGenerator):\n \"\"\"body_centerの座標に基づく移動特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"body_centerが必要\"\"\"\n return self._check_parts_available(['body_center'])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n cx = data['body_center']['x']\n cy = data['body_center']['y']\n \n for w in [5, 15, 30, 60]:\n ws = _scale(w, self.fps)\n roll = dict(min_periods=1, center=True)\n \n # x, y座標の分散の合計\n X[f'cxy_s{w}'] = (\n cx.rolling(ws, **roll).var() \n + \n cy.rolling(ws, **roll).var()\n )\n \n # x, yの範囲の距離\n range_x = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n range_y = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n X[f'sxy_d{w}'] = calculate_distance((range_x, range_y))\n \n # 変位（displacement）\n X[f'disp{w}'] = calculate_distance((\n cx.diff().rolling(ws, min_periods=1).sum(),\n cy.diff().rolling(ws, min_periods=1).sum()\n ))\n \n # 活動性（activity）\n X[f'act{w}'] = calculate_distance((\n cx.diff().rolling(ws, min_periods=1).var(),\n cy.diff().rolling(ws, min_periods=1).var()\n ))\n \n return X\n\nclass CurvatureFeaturesGenerator(BaseGenerator):\n \"\"\"曲率（curvature）と回転率（turn_rate）特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"body_centerが必要\"\"\"\n return self._check_parts_available(['body_center'])\n \n def _calculate_curvature(self, vel_x: pd.Series, vel_y: pd.Series) -> pd.Series:\n \"\"\"\n 速度ベクトルから曲率を計算\n \n Parameters\n ----------\n vel_x : pd.Series\n x方向の速度\n vel_y : pd.Series\n y方向の速度\n \n Returns\n -------\n pd.Series\n 曲率 (curvature): |v × a| / |v|^3\n \"\"\"\n # 加速度\n acc_x = vel_x.diff()\n acc_y = vel_y.diff()\n \n # 外積と速度の大きさ\n cross_prod = vel_x * acc_y - vel_y * acc_x\n vel_mag = calculate_distance((vel_x, vel_y))\n \n # 曲率\n return np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n \n def _calculate_turn_rate(self, vel_x: pd.Series, vel_y: pd.Series) -> pd.Series:\n \"\"\"\n 速度ベクトルから回転率を計算\n \n Parameters\n ----------\n vel_x : pd.Series\n x方向の速度\n vel_y : pd.Series\n y方向の速度\n \n Returns\n -------\n pd.Series\n 回転率 (turn rate): 角度変化の累積\n \"\"\"\n angle = np.arctan2(vel_y, vel_x)\n angle_change = np.abs(angle.diff())\n return angle_change\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n cx = data['body_center']['x']\n cy = data['body_center']['y']\n \n # 速度\n vel_x = cx.diff()\n vel_y = cy.diff()\n \n # 曲率の平均値\n curvature = self._calculate_curvature(vel_x, vel_y)\n for w in [25, 50, 75]:\n ws = _scale(w, self.fps)\n X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 5)).mean()\n \n # 回転率\n angle_change = self._calculate_turn_rate(vel_x, vel_y)\n ws = _scale(30, self.fps)\n X['turn_rate_30'] = angle_change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n \n return X\n\n\nclass MultiscaleFeaturesGenerator(BaseGenerator):\n \"\"\"マルチスケールでの速度統計特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"body_centerが必要\"\"\"\n return self._check_parts_available(['body_center'])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n cx = data['body_center']['x']\n cy = data['body_center']['y']\n \n # 速度（実速度: distance * fps）\n speed = calculate_distance((cx.diff(), cy.diff())) * float(self.fps)\n \n # 複数スケールでの統計量\n scales = [20, 40, 60, 80]\n for scale in scales:\n ws = _scale(scale, self.fps)\n if len(speed) >= ws:\n X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n \n # 短期/長期速度比\n if len(scales) >= 2:\n X['sp_ratio'] = X[f'sp_m{scales[0]}'] / (X[f'sp_m{scales[-1]}'] + 1e-6)\n \n return X\n\nclass StateFeaturesGenerator(BaseGenerator):\n \"\"\"速度状態に基づく特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"body_centerが必要\"\"\"\n return self._check_parts_available(['body_center'])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n cx = data['body_center']['x']\n cy = data['body_center']['y']\n \n # 速度\n speed = calculate_distance((cx.diff(), cy.diff())) * float(self.fps)\n \n # 速度の移動平均\n w_ma = _scale(15, self.fps)\n speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n \n try:\n # 速度状態のビン分割: 0=静止, 1=低速, 2=中速, 3=高速\n bins = [-np.inf, 0.5 * self.fps, 2.0 * self.fps, 5.0 * self.fps, np.inf]\n speed_states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n \n # 各ウィンドウで各状態の割合と状態遷移数を計算\n for window in [20, 40, 60, 80]:\n ws = _scale(window, self.fps)\n if len(speed_states) >= ws:\n # 各状態の割合\n for state in [0, 1, 2, 3]:\n X[f's{state}_{window}'] = (\n (speed_states == state).astype(float)\n .rolling(ws, min_periods=max(1, ws // 5)).mean()\n )\n \n # 状態遷移数\n state_changes = (speed_states != speed_states.shift(1)).astype(float)\n X[f'trans_{window}'] = state_changes.rolling(ws, min_periods=max(1, ws // 5)).sum()\n except Exception:\n # ビン分割に失敗した場合はスキップ\n pass\n \n return X\n\n\nclass LongrangeFeaturesGenerator(BaseGenerator):\n \"\"\"長期ウィンドウでの位置と速度の特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"body_centerが必要\"\"\"\n return self._check_parts_available(['body_center'])\n \n def generate(self, X: pd.DataFrame, data: pd.DataFrame) -> pd.DataFrame:\n cx = data['body_center']['x']\n cy = data['body_center']['y']\n \n # 長期ウィンドウでの平均位置からの距離\n for window in [30, 60, 120]:\n ws = _scale(window, self.fps)\n if len(cx) >= ws:\n X[f'xy_ml{window}'] = calculate_distance((\n cx - cx.rolling(ws, min_periods=max(5, ws // 6)).mean(),\n cy - cy.rolling(ws, min_periods=max(5, ws // 6)).mean()\n ))\n \n # 指数加重移動平均位置からの距離\n for span in [30, 60, 120]:\n s = _scale(span, self.fps)\n X[f'xy_e{span}'] = calculate_distance((\n cx - cx.ewm(span=s, min_periods=1).mean(),\n cy - cy.ewm(span=s, min_periods=1).mean()\n ))\n \n # 速度のパーセンタイルランク\n speed = calculate_distance((cx.diff(), cy.diff())) * float(self.fps)\n for window in [30, 60, 120]:\n ws = _scale(window, self.fps)\n if len(speed) < ws:\n continue\n X[f'sp_pct{window}'] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n \n return X","metadata":{"_uuid":"305d284e-40e0-4494-be7e-c59a17952a97","_cell_guid":"3dcc3cb4-34c9-42c8-bc0b-b4b3597a0b46","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:46.442518Z","iopub.execute_input":"2025-12-14T16:49:46.442868Z","iopub.status.idle":"2025-12-14T16:49:46.499205Z","shell.execute_reply.started":"2025-12-14T16:49:46.442844Z","shell.execute_reply":"2025-12-14T16:49:46.497911Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class PairIntraMouseDistanceGenerator(BaseGenerator):\n \"\"\"各マウス内のbody_part間距離を生成（A, B それぞれ）\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n # avail_B = data['B'].columns.get_level_values(0)\n \n # マウスA内の距離\n distances_A = self._generate_intra_mouse_distances(data['A'], avail_A, prefix='')\n X = pd.concat([X, distances_A], axis=1)\n \n # # マウスB内の距離\n # distances_B = self._generate_intra_mouse_distances(data['B'], avail_B, prefix='B_')\n # X = pd.concat([X, distances_B], axis=1)\n \n return X\n\nclass PairDistanceGenerator(BaseGenerator):\n \"\"\"マウス間の全point間の距離を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n distances = pd.DataFrame({\n f\"12+{p1}+{p2}\": calculate_distance(data['A'][p1], data['B'][p2], squared=True)\n for p1, p2 in itertools.product(self.body_parts_tracked, repeat=2)\n if p1 in avail_A and p2 in avail_B\n })\n \n distances = distances.reindex(\n columns=[f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(self.body_parts_tracked, repeat=2)],\n copy=False\n )\n \n return pd.concat([X, distances], axis=1)\n\nclass PairSpeedGenerator(BaseGenerator):\n \"\"\"各マウスの全身体部位の速度を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n # avail_B = data['B'].columns.get_level_values(0)\n lag = self._get_lag(10)\n \n for part in [\"body_center\", \"ear_left\", \"ear_right\", \"tail_base\"]:\n if part in avail_A:\n X[f'sp_A_{part}'] = calculate_point_speed(data['A'][part], lag, squared=True)\n # if part in avail_B:\n # X[f'sp_B_{part}'] = calculate_point_speed(data['B'][part], lag, squared=True)\n \n return X\n\n\nclass PairAccelerationGenerator(BaseGenerator):\n \"\"\"各マウスの全身体部位の加速度を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n # avail_B = data['B'].columns.get_level_values(0)\n lag = self._get_lag(10)\n \n for part in [\"body_center\", \"ear_left\", \"ear_right\", \"tail_base\"]:\n if part in avail_A:\n X[f'acc_A_{part}'] = calculate_point_acceleration(data['A'][part], lag, squared=True)\n # if part in avail_B:\n # X[f'acc_B_{part}'] = calculate_point_acceleration(data['B'][part], lag, squared=True)\n \n return X\n\nclass PairDistanceLagGenerator(BaseGenerator):\n \"\"\"マウス間距離特徴量に対してラグと差分を追加\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n # distance_columns = [\n # f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(self.body_parts_tracked, repeat=2)\n # if f\"12+{p1}+{p2}\" in X.columns\n # ]\n distance_columns = [f\"12+{p1}+{p1}\" for p1 in [\"body_center\", \"ear_left\", \"ear_right\", \"tail_base\"]\n if f\"12+{p1}+{p1}\" in X.columns]\n \n for col in distance_columns:\n for lag in self.lags:\n l = _scale(lag, self.fps)\n X[f'{col}_lg{lag}'] = X[col].shift(l)\n X[f'{col}_df{lag}'] = X[col] - X[col].shift(l)\n \n return X\n\nclass InterMouseDistanceVelocityGenerator(BaseGenerator):\n \"\"\"マウス間距離の速度と加速度を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n lag = self._get_lag(10)\n \n for part in self.inter_mouse_parts:\n if not self._check_parts_available([part], avail_A, avail_B):\n continue\n dist = calculate_distance(data['A'][part], data['B'][part])\n X[f'dist_vel_{part}'] = calculate_distance_velocity(dist, lag)\n X[f'dist_acc_{part}'] = calculate_distance_acceleration(dist, lag)\n \n return X\n\nclass PairElongationGenerator(BaseGenerator):\n \"\"\"各マウスの伸縮率特徴量を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n # avail_B = data['B'].columns.get_level_values(0)\n \n # マウスAのelongation\n if self._check_parts_available(['nose', 'tail_base', 'ear_left', 'ear_right'], avail_A):\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n \n # # マウスBのelongation\n # if self._check_parts_available(['nose', 'tail_base', 'ear_left', 'ear_right'], avail_B):\n # X['B_elong'] = X['B_nose+tail_base'] / (X['B_ear_left+ear_right'] + 1e-6)\n \n return X\n\n\nclass RelativeOrientationGenerator(BaseGenerator):\n \"\"\"相対的な向き特徴量を生成\"\"\"\n \n def validate(self) -> bool:\n \"\"\"\n 両マウスにnoseとtail_baseが必要\n \n Notes\n -----\n Pair用のGeneratorは、available_body_partsが両マウスの統合のため、\n 正確な検証はできない。少なくともnoseとtail_baseが存在すればTrueを返す。\n \"\"\"\n return self._check_parts_available(['nose', 'tail_base'])\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスに必要な部位が存在する場合のみ生成\n if self._check_parts_available(['nose', 'tail_base'], avail_A, avail_B):\n dir_A = data['A']['nose'] - data['A']['tail_base']\n dir_B = data['B']['nose'] - data['B']['tail_base']\n X['rel_ori'] = calculate_angle_cosine(dir_A, dir_B)\n \n return X\n\nclass ApproachFeatureGenerator(BaseGenerator):\n \"\"\"接近特徴量を生成（近づいているか離れているか）\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにnoseがある場合のみ生成\n if self._check_parts_available(['nose'], avail_A, avail_B):\n # 現在のnose間距離\n cur = calculate_distance(data['A']['nose'], data['B']['nose'], squared=True)\n \n # 過去のnose間距離\n lag = self._get_lag(10)\n shA_n = data['A']['nose'].shift(lag)\n shB_n = data['B']['nose'].shift(lag)\n past = calculate_distance(shA_n, shB_n, squared=True)\n \n # 接近度: 負なら近づいている、正なら離れている\n X['appr'] = cur - past\n \n return X\n\nclass InterMouseStateFeaturesGenerator(BaseGenerator):\n \"\"\"マウス間の距離に基づく状態特徴量を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにbody_centerがある場合のみ生成\n if self._check_parts_available(['body_center'], avail_A, avail_B):\n cd = calculate_distance(data['A']['body_center'], data['B']['body_center'])\n \n # 距離に基づく状態分類\n X['v_cls'] = (cd < 5.0).astype(float) # very close\n X['cls'] = ((cd >= 5.0) & (cd < 15.0)).astype(float) # close\n X['med'] = ((cd >= 15.0) & (cd < 30.0)).astype(float) # medium\n X['far'] = (cd >= 30.0).astype(float) # far\n \n return X\n\nclass InterMouseCoordFeaturesGenerator(BaseGenerator):\n \"\"\"マウス間の距離統計量と座標特徴量を生成\"\"\"\n \n def _calculate_velocity_dot_product(\n self,\n data_A: pd.DataFrame,\n data_B: pd.DataFrame,\n part: str = 'body_center'\n ) -> pd.Series:\n \"\"\"\n 2つのマウスの速度ベクトルの内積を計算\n \n Parameters\n ----------\n data_A : pd.DataFrame\n マウスAのデータ\n data_B : pd.DataFrame\n マウスBのデータ\n part : str, optional\n 計算対象の身体部位、デフォルトは'body_center'\n \n Returns\n -------\n pd.Series\n 速度ベクトルの内積\n \"\"\"\n A_vx = data_A[part]['x'].diff()\n A_vy = data_A[part]['y'].diff()\n B_vx = data_B[part]['x'].diff()\n B_vy = data_B[part]['y'].diff()\n \n return A_vx * B_vx + A_vy * B_vy\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにbody_centerがある場合のみ生成\n if self._check_parts_available(['body_center'], avail_A, avail_B):\n cd_full = calculate_distance(data['A']['body_center'], data['B']['body_center'], squared=True)\n \n for w in [5, 15, 30, 60]:\n ws = _scale(w, self.fps)\n roll = dict(min_periods=1, center=True)\n \n # 距離の統計量\n X[f'd_m{w}'] = cd_full.rolling(ws, **roll).mean()\n X[f'd_s{w}'] = cd_full.rolling(ws, **roll).std()\n X[f'd_mn{w}'] = cd_full.rolling(ws, **roll).min()\n X[f'd_mx{w}'] = cd_full.rolling(ws, **roll).max()\n \n # 相互作用特徴量（距離の分散の逆数）\n d_var = cd_full.rolling(ws, **roll).var()\n X[f'int{w}'] = 1 / (1 + d_var)\n \n # 座標（速度の内積）特徴量\n coord = self._calculate_velocity_dot_product(data['A'], data['B'], 'body_center')\n X[f'co_m{w}'] = coord.rolling(ws, **roll).mean()\n X[f'co_s{w}'] = coord.rolling(ws, **roll).std()\n \n return X\n\nclass InterMouseProximityFeaturesGenerator(BaseGenerator):\n \"\"\"マウス間の近接確率特徴量を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにnoseがある場合のみ生成\n if 'nose' in avail_A and 'nose' in avail_B:\n nn = calculate_distance(data['A']['nose'], data['B']['nose'])\n \n # 近接閾値（距離 < 10.0）に基づく確率特徴量\n for lag in [10, 20, 40]:\n l = _scale(lag, self.fps)\n is_cl = (nn < 10.0).astype(float)\n X[f'cl_ps{lag}'] = is_cl.rolling(l, min_periods=1).mean()\n \n return X\n\n\nclass InterMouseVelocityAngleFeaturesGenerator(BaseGenerator):\n \"\"\"マウス間の速度角度特徴量を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにbody_centerがある場合のみ生成\n if ('body_center' in avail_A) and ('body_center' in avail_B):\n # 各マウスの速度ベクトル\n Avx = data['A']['body_center']['x'].diff()\n Avy = data['A']['body_center']['y'].diff()\n Bvx = data['B']['body_center']['x'].diff()\n Bvy = data['B']['body_center']['y'].diff()\n \n # 速度ベクトル間の角度（コサイン類似度）\n val = calculate_angle_cosine((Avx, Avy), (Bvx, Bvy))\n \n # 複数オフセットでシフト\n for off in [-60, -30, 30, 60]:\n o = _scale_signed(off, self.fps)\n X[f'va_{off}'] = val.shift(-o)\n \n # 距離の変動係数\n cd_full = calculate_distance(data['A']['body_center'], data['B']['body_center'], squared=True)\n X['int_con'] = calculate_coefficient_of_variation(cd_full, window=30, fps=self.fps)\n \n return X\n\n\nclass InterMouseInteractionFeaturesGenerator(BaseGenerator):\n \"\"\"マウス間の相互作用特徴量（leading indicator）を生成\"\"\"\n \n def generate(self, X: pd.DataFrame, data: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n avail_A = data['A'].columns.get_level_values(0)\n avail_B = data['B'].columns.get_level_values(0)\n \n # 両マウスにbody_centerがある場合のみ生成\n if ('body_center' not in avail_A) or ('body_center' not in avail_B):\n return X\n \n # 相対位置ベクトル（A -> B）\n rel_x = data['A']['body_center']['x'] - data['B']['body_center']['x']\n rel_y = data['A']['body_center']['y'] - data['B']['body_center']['y']\n rel_dist = calculate_distance((rel_x, rel_y))\n \n # 各マウスの速度ベクトル\n A_vx = data['A']['body_center']['x'].diff()\n A_vy = data['A']['body_center']['y'].diff()\n B_vx = data['B']['body_center']['x'].diff()\n B_vy = data['B']['body_center']['y'].diff()\n \n # Leading指標: 速度ベクトルと相対位置ベクトルの内積 / (速度 * 距離)\n # Aが相手に向かって動いているか\n A_lead = (A_vx * rel_x + A_vy * rel_y) / (calculate_distance((A_vx, A_vy)) * rel_dist + 1e-6)\n # Bが相手に向かって動いているか\n B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (calculate_distance((B_vx, B_vy)) * rel_dist + 1e-6)\n\n\n\n\n \n # 複数ウィンドウでの平均\n for window in [30, 60]:\n ws = _scale(window, self.fps)\n X[f'A_ld{window}'] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n X[f'B_ld{window}'] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n approach = -rel_dist.diff()\n chase = approach * B_lead\n w = 30\n ws = _scale(w, self.fps)\n X[f'chase_{w}'] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n for window in [60, 120]:\n ws = _scale(window, self.fps)\n A_sp = calculate_distance((A_vx, A_vy))\n B_sp = calculate_distance((B_vx, B_vy))\n X[f'sp_cor{window}'] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n \n return X","metadata":{"_uuid":"69d1d934-a765-46a4-8bb5-7e28c094cb56","_cell_guid":"b9afb4ea-3a6d-4470-9006-3f49ad648483","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:46.623525Z","iopub.execute_input":"2025-12-14T16:49:46.623869Z","iopub.status.idle":"2025-12-14T16:49:46.672419Z","shell.execute_reply.started":"2025-12-14T16:49:46.623847Z","shell.execute_reply":"2025-12-14T16:49:46.671387Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ====================\n# FeatureStore\n# ====================\n\nclass SingleFeatureStore:\n \"\"\"単一マウスの特徴量ストア\"\"\"\n \n def __init__(self, feature_config: Dict, body_parts_tracked: List[str], fps: float):\n \"\"\"\n Parameters\n ----------\n feature_config : Dict\n 特徴量設定\n body_parts_tracked : List[str]\n 追跡する身体部位のリスト\n fps : float\n フレームレート\n \"\"\"\n self.feature_config = feature_config\n self.feature_config['body_parts_tracked'] = body_parts_tracked\n self.feature_config['lags'] = feature_config.get('lags', [10, 20, 40])\n self.fps = fps\n self.body_parts_tracked = body_parts_tracked\n \n def generate(self, single_mouse: pd.DataFrame) -> pd.DataFrame:\n \"\"\"\n 単一マウスの特徴量を生成\n \n Parameters\n ----------\n single_mouse : pd.DataFrame\n 単一マウスのデータ（multi_index: [body_part, x_or_y]）\n \n Returns\n -------\n pd.DataFrame\n 生成された特徴量\n \"\"\"\n available_body_parts = single_mouse.columns.get_level_values(0)\n X = pd.DataFrame(index=single_mouse.index)\n \n # Generatorを順番に実行\n generators = [\n DistanceGenerator(self.feature_config, available_body_parts, self.fps),\n SpeedGenerator(self.feature_config, available_body_parts, self.fps),\n AccelerationGenerator(self.feature_config, available_body_parts, self.fps),\n CrossSpeedGenerator(self.feature_config, available_body_parts, self.fps),\n DistanceLagGenerator(self.feature_config, available_body_parts, self.fps),\n ElongationGenerator(self.feature_config, available_body_parts, self.fps),\n BodyAngleGenerator(self.feature_config, available_body_parts, self.fps),\n BodyCenterMovementGenerator(self.feature_config, available_body_parts, self.fps),\n CurvatureFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n MultiscaleFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n StateFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n LongrangeFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n LeftRightPairGenerator(self.feature_config, available_body_parts, self.fps),\n ]\n \n for generator in generators:\n if generator.validate():\n X = generator.generate(X, single_mouse)\n \n return X.astype(np.float32, copy=False)\n\n\nclass PairFeatureStore:\n \"\"\"ペアマウスの特徴量ストア\"\"\"\n \n def __init__(self, feature_config: Dict, body_parts_tracked: List[str], fps: float):\n \"\"\"\n Parameters\n ----------\n feature_config : Dict\n 特徴量設定\n body_parts_tracked : List[str]\n 追跡する身体部位のリスト\n fps : float\n フレームレート\n \"\"\"\n self.feature_config = feature_config\n self.feature_config['body_parts_tracked'] = body_parts_tracked\n self.feature_config['lags'] = feature_config.get('lags', [10, 20, 40])\n self.fps = fps\n self.body_parts_tracked = body_parts_tracked\n \n def generate(self, mouse_pair: pd.DataFrame) -> pd.DataFrame:\n \"\"\"\n ペアマウスの特徴量を生成\n \n Parameters\n ----------\n mouse_pair : pd.DataFrame\n ペアマウスのデータ（multi_index: ['A'/'B', body_part, x_or_y]）\n \n Returns\n -------\n pd.DataFrame\n 生成された特徴量\n \"\"\"\n avail_A = mouse_pair['A'].columns.get_level_values(0)\n avail_B = mouse_pair['B'].columns.get_level_values(0)\n \n # 便宜上、両方の利用可能部位を統合（生成時に各Generatorで判定）\n available_body_parts = avail_A.union(avail_B).unique()\n \n X = pd.DataFrame(index=mouse_pair.index)\n data = {'A': mouse_pair['A'], 'B': mouse_pair['B']}\n \n # Generatorを順番に実行\n generators = [\n PairIntraMouseDistanceGenerator(self.feature_config, available_body_parts, self.fps),\n PairDistanceGenerator(self.feature_config, available_body_parts, self.fps),\n PairSpeedGenerator(self.feature_config, available_body_parts, self.fps),\n PairAccelerationGenerator(self.feature_config, available_body_parts, self.fps),\n PairDistanceLagGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseDistanceVelocityGenerator(self.feature_config, available_body_parts, self.fps),\n PairElongationGenerator(self.feature_config, available_body_parts, self.fps),\n RelativeOrientationGenerator(self.feature_config, available_body_parts, self.fps),\n ApproachFeatureGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseStateFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseCoordFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseProximityFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseVelocityAngleFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n InterMouseInteractionFeaturesGenerator(self.feature_config, available_body_parts, self.fps),\n ]\n \n for generator in generators:\n if generator.validate():\n X = generator.generate(X, data)\n \n return X.astype(np.float32, copy=False)","metadata":{"_uuid":"4b4ba977-1c16-4c95-9b83-13e3b418248d","_cell_guid":"0b875049-2938-419e-a136-5dee76746847","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:46.761056Z","iopub.execute_input":"2025-12-14T16:49:46.761909Z","iopub.status.idle":"2025-12-14T16:49:46.778753Z","shell.execute_reply.started":"2025-12-14T16:49:46.761877Z","shell.execute_reply":"2025-12-14T16:49:46.777607Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training","metadata":{"_uuid":"52737390-d986-44e5-8a6a-2ca7c4dbd8fe","_cell_guid":"b7f0bf98-72b4-4569-b2ca-b58e8e5188aa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def progress_callback(study, trial): \n # 現在のベスト結果を表示\n if study.best_trial.number == trial.number:\n print(f\"=== Trial {trial.number} - Current Best ===\")\n print(f\"Best F1 Score: {study.best_value:.4f}\")\n print(f\"Best Params: {study.best_params}\")\n print(\"=\" * 50)\n\n\ndef apply_post_processing(predictions, thres, W, M):\n \"\"\"\n 後処理を適用する関数\n \n Args:\n predictions: 二値化された予測結果 (0 or 1)\n W: ギャップフィリング - 1と予測された行の間がW以下の場合、その間を1にする\n M: ノイズ除去 - 1が連続する数がM以下の場合0にする\n \n Returns:\n 後処理後の予測結果\n \"\"\"\n def process_group(group, W, M):\n \"\"\"各グループ（主キー）内でW・M処理を実行\"\"\"\n # スコアをソート（時系列順に並んでいることを前提）\n predictions = group[\"pred\"].copy()\n \n # ★★★ W処理：ギャップフィリング ★★★\n if W > 0:\n ones_indices = np.where(predictions == 1)[0]\n if len(ones_indices) > 1:\n for i in range(len(ones_indices) - 1):\n gap_start = ones_indices[i] + 1\n gap_end = ones_indices[i + 1]\n gap_length = gap_end - gap_start\n if gap_length <= W:\n predictions[gap_start:gap_end] = 1\n \n # ★★★ M処理：短い連続除去 ★★★\n if M > 0:\n diff = np.diff(np.concatenate(([0], predictions, [0])))\n starts = np.where(diff == 1)[0]\n ends = np.where(diff == -1)[0]\n for start, end in zip(starts, ends):\n length = end - start\n if length <= M:\n predictions[start:end] = 0\n \n # 結果を追加\n result = group.copy()\n result['final_pred'] = predictions\n return result\n \n predictions = predictions.copy()\n predictions[\"pred\"] = (predictions[\"pred\"] >= thres).astype(int)\n processed = predictions.groupby([\"video_id\", \"agent_id\", \"target_id\"]).apply(lambda x: process_group(x, W, M))\n return processed['final_pred'].values\n\n\ndef create_predictions_with_postprocessing(pred_probs, video_ids, agent_ids, target_ids, threshold_params):\n \"\"\"\n 予測確率から後処理を適用した最終予測を作成する共通関数\n \n Args:\n pred_probs: 予測確率（0-1の値）\n video_ids: video_idの配列\n agent_ids: agent_idの配列\n target_ids: target_idの配列\n threshold_params: {\"threshold\": float, \"W\": int, \"M\": int}の辞書\n \n Returns:\n 後処理後の予測値（0-1のマスク値）\n \"\"\"\n predictions_df = pd.DataFrame({\n \"video_id\": video_ids,\n \"agent_id\": agent_ids,\n \"target_id\": target_ids,\n \"pred\": pred_probs\n })\n \n post_processed = apply_post_processing(\n predictions_df, \n threshold_params[\"threshold\"], \n threshold_params[\"W\"], \n threshold_params[\"M\"]\n )\n \n return post_processed\n\n\ndef tune_threshold(oof_action, y_action, video_ids, agent_ids, target_ids):\n def objective(trial):\n # パラメータをサジェスト\n threshold = trial.suggest_float(\"threshold\", 0, 1, step=0.01)\n W = trial.suggest_int(\"W\", 0, 60) # ギャップフィリング範囲\n M = trial.suggest_int(\"M\", 0, 30) # 短い連続除去閾値\n\n predictions_df = pd.DataFrame({\n \"video_id\": video_ids,\n \"agent_id\": agent_ids,\n \"target_id\": target_ids,\n \"pred\": oof_action\n })\n \n # 後処理を適用\n processed_pred = apply_post_processing(predictions_df, threshold, W, M)\n \n # F1スコアを計算\n return f1_score(y_action, processed_pred, zero_division=0)\n \n # 最適化実行\n study = optuna.create_study(direction=\"maximize\")\n\n initial_values = [\n {\"threshold\": 0.5, \"W\": 0, \"M\": 0},\n {\"threshold\": 0.3, \"W\": 0, \"M\": 0},\n {\"threshold\": 0.7, \"W\": 0, \"M\": 0}\n ]\n for params in initial_values:\n study.enqueue_trial(params)\n study.optimize(objective, n_trials=200, n_jobs=1, callbacks=[progress_callback]) # 試行回数を増やす\n \n return study.best_params","metadata":{"_uuid":"7732615e-84ec-46ec-834e-360c70de5b55","_cell_guid":"01ebc65b-e73b-4fba-90f5-62b15b1ca556","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:47.082351Z","iopub.execute_input":"2025-12-14T16:49:47.082738Z","iopub.status.idle":"2025-12-14T16:49:47.100753Z","shell.execute_reply.started":"2025-12-14T16:49:47.082714Z","shell.execute_reply":"2025-12-14T16:49:47.09962Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prediction","metadata":{"_uuid":"075dce26-7803-41dd-9306-552149526eda","_cell_guid":"2296c30d-e9c0-44b5-9f1d-ded639ce2879","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"class ProcessedData(NamedTuple):\n \"\"\"Processed data containing features, labels, and metadata.\"\"\"\n features: pd.DataFrame\n labels: Optional[pd.DataFrame]\n metadata: pd.DataFrame\n\ndef reduce_mem_usage(train_data):\n \"\"\" iterate through all the columns of a dataframe and modify the data type\n to reduce memory usage.\n \"\"\"\n start_mem = train_data.memory_usage().sum() / 1024**2\n print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n \n for col in train_data.columns:\n col_type = train_data[col].dtype\n\n if col_type != object:\n c_min = train_data[col].min()\n c_max = train_data[col].max()\n if str(col_type)[:3] == 'int':\n if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n train_data[col] = train_data[col].astype(np.int8)\n elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n train_data[col] = train_data[col].astype(np.int16)\n elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n train_data[col] = train_data[col].astype(np.int32)\n elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n train_data[col] = train_data[col].astype(np.int64) \n else:\n if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n train_data[col] = train_data[col].astype(np.float16)\n elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n train_data[col] = train_data[col].astype(np.float32)\n else:\n train_data[col] = train_data[col].astype(np.float64)\n else:\n train_data[col] = train_data[col].astype('category')\n\n end_mem = train_data.memory_usage().sum() / 1024**2\n print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n return train_data\n\n\ndef generate_features_and_concat(behavior_data_list: List[BehaviorData]) -> ProcessedData:\n \"\"\"\n Generate features from behavior data list and concatenate results.\n \n Args:\n behavior_data_list: List of BehaviorData instances\n \n Returns:\n ProcessedData containing:\n - features: Concatenated features DataFrame\n - labels: Concatenated labels DataFrame (None in test mode)\n - metadata: Concatenated metadata DataFrame\n \"\"\"\n # Generate features for all behavior data\n feats_parts = []\n for behavior_data in tqdm(behavior_data_list):\n feats_parts.append(behavior_data.generate_features())\n gc.collect()\n \n # Concatenate features and reduce memory usage\n X = pd.concat(feats_parts, axis=0, ignore_index=True)\n X = reduce_mem_usage(X)\n \n # Concatenate labels if they exist (train mode), otherwise return None (test mode)\n if behavior_data_list[0].labels is not None:\n labels = pd.concat([data.labels for data in behavior_data_list], axis=0, ignore_index=True)\n else:\n labels = None\n \n # Concatenate metadata\n metadata = pd.concat([data.metadata for data in behavior_data_list], axis=0, ignore_index=True)\n \n # Clean up\n del feats_parts\n gc.collect()\n \n return ProcessedData(features=X, labels=labels, metadata=metadata)","metadata":{"_uuid":"54fad954-c901-460d-9d10-b6ea3075add1","_cell_guid":"7ec39945-0ae2-438f-ac2c-6a7c6df734e3","trusted":true,"collapsed":false,"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-12-14T16:49:47.422947Z","iopub.execute_input":"2025-12-14T16:49:47.423654Z","iopub.status.idle":"2025-12-14T16:49:47.439113Z","shell.execute_reply.started":"2025-12-14T16:49:47.423627Z","shell.execute_reply":"2025-12-14T16:49:47.438122Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"F Beta customized for the data format of the MABe challenge.\"\"\"\n\nimport json\n\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\n\n\nclass HostVisibleError(Exception):\n pass\n\n\ndef single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n label_frames: defaultdict[str, set[int]] = defaultdict(set)\n prediction_frames: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_solution.to_dicts():\n label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n for video in lab_solution['video_id'].unique():\n active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first() # ty: ignore\n active_labels: set[str] = set(json.loads(active_labels))\n predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_submission.filter(pl.col('video_id') == video).to_dicts():\n # Since the labels are sparse, we can't evaluate prediction keys not in the active labels.\n if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n continue\n\n new_frames = set(range(row['start_frame'], row['stop_frame']))\n # Ignore truly redundant predictions.\n new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n # A single agent can have multiple targets per frame (ex: evading all other mice) but only one action per target per frame.\n raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n prediction_frames[row['prediction_key']].update(new_frames)\n predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n tps = defaultdict(int)\n fns = defaultdict(int)\n fps = defaultdict(int)\n for key, pred_frames in prediction_frames.items():\n action = key.split('_')[-1]\n matched_label_frames = label_frames[key]\n tps[action] += len(pred_frames.intersection(matched_label_frames))\n fns[action] += len(matched_label_frames.difference(pred_frames))\n fps[action] += len(pred_frames.difference(matched_label_frames))\n\n distinct_actions = set()\n for key, frames in label_frames.items():\n action = key.split('_')[-1]\n distinct_actions.add(action)\n if key not in prediction_frames:\n fns[action] += len(frames)\n\n action_f1s = []\n for action in distinct_actions:\n if tps[action] + fns[action] + fps[action] == 0:\n action_f1s.append(0)\n print(f\"{action}'s F1 is 0\")\n else:\n f1 = (1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action])\n action_f1s.append(f1)\n print(f\"{action}'s F1 is {f1}\")\n return sum(action_f1s) / len(action_f1s)\n\n\ndef mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n \"\"\"\n Doctests:\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 0, 'stop_frame': 10}, # Wrong action\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.500000000000'\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.250000000000'\n\n >>> # Overlapping solution events, one prediction matching both.\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 10, 'stop_frame': 20, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 20},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 30, 'stop_frame': 40, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 40},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.6666666666666666\n \"\"\"\n if len(solution) == 0 or len(submission) == 0:\n raise ValueError('Missing solution or submission data')\n\n expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n for col in expected_cols:\n if col not in solution.columns:\n raise ValueError(f'Solution is missing column {col}')\n if col not in submission.columns:\n raise ValueError(f'Submission is missing column {col}')\n\n solution: pl.DataFrame = pl.DataFrame(solution)\n submission: pl.DataFrame = pl.DataFrame(submission)\n assert (solution['start_frame'] <= solution['stop_frame']).all()\n assert (submission['start_frame'] <= submission['stop_frame']).all()\n solution_videos = set(solution['video_id'].unique())\n # Need to align based on video IDs as we can't rely on the row IDs for handling public/private splits.\n submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n solution = solution.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('label_key'),\n )\n submission = submission.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('prediction_key'),\n )\n\n lab_scores = []\n for lab in solution['lab_id'].unique():\n lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n lab_videos = set(lab_solution['video_id'].unique())\n lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n f1 = single_lab_f1(lab_solution, lab_submission, beta=beta)\n print(f\"{lab}'s F1 is {f1}\")\n lab_scores.append(f1)\n\n return sum(lab_scores) / len(lab_scores)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n \"\"\"\n F1 score for the MABe Challenge\n \"\"\"\n solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n return mouse_fbeta(solution, submission, beta=beta)","metadata":{"_uuid":"0e78989d-53e9-4d90-8d11-da21f553b4e1","_cell_guid":"fe18ebb3-de22-4bad-ae84-7ee6a7d20be6","trusted":true,"collapsed":false,"_kg_hide-input":true,"papermill":{"duration":2.648741,"end_time":"2025-10-30T12:14:58.420056","exception":false,"start_time":"2025-10-30T12:14:55.771315","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:47.586065Z","iopub.execute_input":"2025-12-14T16:49:47.586367Z","iopub.status.idle":"2025-12-14T16:49:47.611038Z","shell.execute_reply.started":"2025-12-14T16:49:47.586345Z","shell.execute_reply":"2025-12-14T16:49:47.609888Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"_uuid":"50e9c672-7377-426f-abe8-7849af563168","_cell_guid":"4ae3b2c5-fb82-48c3-bc32-73845dec9bf9","trusted":true,"collapsed":false,"papermill":{"duration":0.005956,"end_time":"2025-10-30T12:15:00.909269","exception":false,"start_time":"2025-10-30T12:15:00.903313","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"data_loader = DataLoader(CFG.root_dir)","metadata":{"_uuid":"22effd4b-c814-43d0-a829-d49f431c4235","_cell_guid":"016bd3aa-4453-4c4a-ad71-5d97768c67b7","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:47.891438Z","iopub.execute_input":"2025-12-14T16:49:47.891809Z","iopub.status.idle":"2025-12-14T16:49:47.897409Z","shell.execute_reply.started":"2025-12-14T16:49:47.891787Z","shell.execute_reply":"2025-12-14T16:49:47.896385Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = data_loader.load_train_csv(exclude_unuse_lab=True)\ntest = data_loader.load_test_csv()\nlab_list = data_loader.lab_list\nif CFG.mode == 'validate':\n solution = data_loader.create_solution_df()","metadata":{"_uuid":"7e80ed85-b5a9-4018-b86e-b78cc05ff997","_cell_guid":"b91a45b2-f7da-4526-8bcb-a95291201bf3","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:48.098295Z","iopub.execute_input":"2025-12-14T16:49:48.099083Z","iopub.status.idle":"2025-12-14T16:49:52.579794Z","shell.execute_reply.started":"2025-12-14T16:49:48.099052Z","shell.execute_reply":"2025-12-14T16:49:52.57868Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training, validation and submission","metadata":{"_uuid":"7d9c2d29-b810-440d-9845-5ae970d792ee","_cell_guid":"2422203e-a019-45a0-b883-7beb1136e1f8","trusted":true,"collapsed":false,"papermill":{"duration":0.006102,"end_time":"2025-10-30T12:15:01.324633","exception":false,"start_time":"2025-10-30T12:15:01.318531","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def robustify(submission, dataset, traintest, traintest_directory=None):\n if traintest_directory is None:\n traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n old_submission = submission.copy()\n submission = submission[submission.start_frame < submission.stop_frame]\n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped frames with start >= stop\")\n \n old_submission = submission.copy()\n group_list = []\n for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n group = group.sort_values('start_frame')\n mask = np.ones(len(group), dtype=bool)\n last_stop_frame = 0\n for i, (_, row) in enumerate(group.iterrows()):\n if row['start_frame'] < last_stop_frame:\n mask[i] = False\n else:\n last_stop_frame = row['stop_frame']\n group_list.append(group[mask])\n \n submission = pd.concat(group_list)\n \n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped duplicate frames\")\n \n s_list = []\n for idx, row in dataset.iterrows():\n lab_id = row['lab_id']\n if lab_id in [\n \"CalMS21_supplemental\",\n \"CalMS21_task1\",\n \"CalMS21_task2\",\n \"MABe22_keypoints\",\n \"MABe22_movies\"\n ]:\n continue\n \n video_id = row['video_id']\n if (submission.video_id == video_id).any():\n continue\n \n if type(row.behaviors_labeled) != str:\n continue\n\n print(f\"Video {video_id} has no predictions.\")\n \n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n vid = pd.read_parquet(path)\n \n vid_behaviors = json.loads(row['behaviors_labeled'])\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n start_frame = vid.video_frame.min()\n stop_frame = vid.video_frame.max() + 1\n \n # for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n # batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n # for i, (_, action_row) in enumerate(actions.iterrows()):\n # batch_start = start_frame + i * batch_length\n # batch_stop = min(batch_start + batch_length, stop_frame)\n # s_list.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n\n if len(s_list) > 0:\n submission = pd.concat([\n submission,\n pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n ])\n print(\"ERROR: Filled empty videos\")\n\n submission = submission.reset_index(drop=True)\n \n return submission","metadata":{"_uuid":"911fabc4-c929-4a72-b57f-5184ab7ca2d7","_cell_guid":"904ed6f4-fb35-4855-be99-2df0625c07ed","trusted":true,"collapsed":false,"_kg_hide-input":false,"papermill":{"duration":0.021495,"end_time":"2025-10-30T12:15:01.352815","exception":false,"start_time":"2025-10-30T12:15:01.33132","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:52.581822Z","iopub.execute_input":"2025-12-14T16:49:52.582092Z","iopub.status.idle":"2025-12-14T16:49:52.594498Z","shell.execute_reply.started":"2025-12-14T16:49:52.582073Z","shell.execute_reply":"2025-12-14T16:49:52.593509Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_multiclass(pred, meta):\n ama = np.argmax(pred.values, axis=1)\n max_proba = pred.max(axis=1).values\n\n ama = np.where(max_proba > 0, ama, -1)\n ama = pd.Series(ama, index=meta.video_frame)\n \n changes_mask = (ama != ama.shift(1)).values\n ama_changes = ama[changes_mask]\n meta_changes = meta[changes_mask]\n \n mask = ama_changes.values >= 0\n mask[-1] = False\n \n submission_part = pd.DataFrame({\n 'video_id': meta_changes['video_id'][mask].values,\n 'agent_id': meta_changes['agent_id'][mask].values,\n 'target_id': meta_changes['target_id'][mask].values,\n 'action': pred.columns[ama_changes[mask].values],\n 'start_frame': ama_changes.index[mask],\n 'stop_frame': ama_changes.index[1:][mask[:-1]]\n })\n \n stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n for i in range(len(submission_part)):\n video_id = submission_part.video_id.iloc[i]\n agent_id = submission_part.agent_id.iloc[i]\n target_id = submission_part.target_id.iloc[i]\n if stop_video_id[i] != video_id or stop_agent_id[i] != agent_id or stop_target_id[i] != target_id:\n new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n\n return submission_part","metadata":{"_uuid":"e7d2d614-22fd-42df-8af6-5f9dc2bdfdc0","_cell_guid":"d758af1c-8c0b-4f8e-8852-c52c1ecc4512","trusted":true,"collapsed":false,"papermill":{"duration":0.0187,"end_time":"2025-10-30T12:15:01.378753","exception":false,"start_time":"2025-10-30T12:15:01.360053","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:52.595815Z","iopub.execute_input":"2025-12-14T16:49:52.59611Z","iopub.status.idle":"2025-12-14T16:49:52.632971Z","shell.execute_reply.started":"2025-12-14T16:49:52.596089Z","shell.execute_reply":"2025-12-14T16:49:52.631845Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_saved_trainer(\n model_path, model_name, section, action, \n threshold_base_path=\"/kaggle/input/mabe-xgb-v7/xgboost\",\n do_modeling=False, do_tuning=False\n):\n \"\"\"\n 保存済みのtrainerモデルとthresholdをロードする\n \n Args:\n model_path: モデルの保存先ディレクトリパス\n model_name: モデル名\n section: セクション名\n action: アクション名\n threshold_base_path: thresholdファイルのベースパス\n do_modeling: Trueの場合、モデルが存在してもNoneを返す（再学習）\n do_tuning: Trueの場合、thresholdが存在してもNoneを返す（再チューニング）\n \n Returns:\n (trainer, threshold) or (None, None) (モデルファイルが見つからない場合、またはdo_modeling/do_tuningがTrueの場合)\n \"\"\"\n trainer = None\n threshold = None\n \n # モデルのロード\n if not do_modeling:\n files = glob.glob(f\"{model_path}/{model_name}/{section}/{action}/*_trainer_*.pkl\")\n if len(files) == 1:\n trainer = joblib.load(files[0])\n \n # thresholdのロード\n if not do_tuning:\n try:\n threshold = joblib.load(f\"{threshold_base_path}/{section}/{action}/threshold.pkl\")\n except FileNotFoundError:\n threshold = None\n \n return trainer, threshold","metadata":{"_uuid":"3bfbc08b-bb45-4d99-9218-0841f5b4b516","_cell_guid":"c4e2f0a0-8de7-4c85-9ebf-47e0cd851c22","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:52.635357Z","iopub.execute_input":"2025-12-14T16:49:52.635785Z","iopub.status.idle":"2025-12-14T16:49:52.656737Z","shell.execute_reply.started":"2025-12-14T16:49:52.635761Z","shell.execute_reply":"2025-12-14T16:49:52.655648Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cross_validate_classifier(data, lab_id, section):\n oof = pd.DataFrame(index=data.metadata.video_frame)\n \n f1_list = []\n submission_list = []\n thresholds = {}\n # print(len(X.columns), X.columns)\n \n for action in data.labels.columns:\n oof[action] = 0\n action_mask = ~ data.labels[action].isna().values\n y_action = data.labels[action][action_mask].values.astype(int)\n X_action = data.features[action_mask]\n groups_action = data.metadata.video_id[action_mask].astype(str) + \"_\" + data.metadata.agent_id[action_mask] + \"_\" + data.metadata.target_id[action_mask]\n video_id = data.metadata.video_id[action_mask] \n agent_id = data.metadata.agent_id[action_mask]\n target_id = data.metadata.target_id[action_mask]\n\n if (y_action == 0).all():\n print(f\"No data for action: {action}\")\n continue\n try:\n null_count = (X_action.isnull().mean().sort_values(ascending=False))\n display(null_count.head(50))\n \n with warnings.catch_warnings():\n warnings.filterwarnings('ignore', category=RuntimeWarning)\n\n trainer, threshold = load_saved_trainer(\n CFG.model_path, CFG.model_name, section, action,\n do_modeling=CFG.do_modeling, do_tuning=CFG.do_tuning\n )\n if trainer is None:\n trainer = Trainer(\n estimator=clone(CFG.model),\n cv=CFG.cv,\n cv_args={\"groups\": groups_action},\n metric=f1_score,\n task=\"binary\",\n verbose=True,\n save=True,\n save_path=f\"{CFG.model_name}/{section}/{action}\"\n )\n\n trainer.fit(X_action, y_action)\n else:\n path = Path(f\"{CFG.model_name}/{section}/{action}\")\n path.mkdir(parents=True, exist_ok=True)\n \n oof_action = trainer.oof_preds\n\n\n # 後処理を適用\n if threshold is None:\n threshold = tune_threshold(oof_action, y_action, video_id, agent_id, target_id)\n thresholds[action] = threshold\n post = create_predictions_with_postprocessing(\n pred_probs=oof_action,\n video_ids=video_id,\n agent_ids=agent_id,\n target_ids=target_id,\n threshold_params=threshold\n )\n \n f1 = f1_score(y_action, post, zero_division=0)\n f1_list.append((lab_id, action, f1))\n\n joblib.dump(oof_action, f\"{CFG.model_name}/{section}/{action}/oof_pred_probs.pkl\")\n joblib.dump(threshold, f\"{CFG.model_name}/{section}/{action}/threshold.pkl\")\n \n print(f\"\\tF1: {f1:.4f} Section: {section} Action: {action}\")\n\n del trainer\n gc.collect()\n\n except Exception as e:\n print(f\"Error for action: {action} - {e}\")\n continue\n \n \n oof_column = np.zeros(len(data.labels))\n oof_column[action_mask] = oof_action * post\n oof[action] = oof_column\n\n del oof_action, action_mask, X_action, y_action, groups_action\n gc.collect()\n\n if len(oof.columns) > 0:\n submission_part = predict_multiclass(oof, data.metadata)\n submission_list.append(submission_part) \n \n return submission_list, f1_list, thresholds","metadata":{"_uuid":"6d4a0757-277d-44ed-92b2-354ce896bf69","_cell_guid":"8b4fa924-4bbb-4fc4-9eaf-5f858782c58f","trusted":true,"collapsed":false,"papermill":{"duration":0.019185,"end_time":"2025-10-30T12:15:01.425576","exception":false,"start_time":"2025-10-30T12:15:01.406391","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:52.658026Z","iopub.execute_input":"2025-12-14T16:49:52.658466Z","iopub.status.idle":"2025-12-14T16:49:52.68375Z","shell.execute_reply.started":"2025-12-14T16:49:52.658442Z","shell.execute_reply":"2025-12-14T16:49:52.6826Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"82929c61-eca2-4a92-9447-2a1370524e58","_cell_guid":"873e0719-6918-48d8-8a32-3ac8368b5cc1","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(body_parts_tracked, switch_tr, section, lab_id): \n test_subset = test[test.lab_id == lab_id]\n test_preprocessor = Preprocessor(data_loader, mode='test', dataset=test_subset, body_parts_tracked=body_parts_tracked)\n generator = test_preprocessor.generate_mouse_data(\n generate_single=(switch_tr == 'single'), \n generate_pair=(switch_tr == 'pair')\n )\n\n submission_list = []\n for behavior_data in tqdm(generator):\n assert behavior_data.behavior_type == switch_tr\n try:\n X_te = behavior_data.generate_features()\n gc.collect()\n \n pred = pd.DataFrame(index=behavior_data.metadata.video_frame)\n for action in behavior_data.actions:\n trainer, threshold = load_saved_trainer(\n CFG.model_path, CFG.model_name, section, action,\n do_modeling=False, do_tuning=False\n )\n if trainer is not None:\n _pred = trainer.predict(X_te)\n \n # 後処理を適用\n post = create_predictions_with_postprocessing(\n pred_probs=_pred,\n video_ids=behavior_data.metadata.video_id.values,\n agent_ids=behavior_data.metadata.agent_id.values,\n target_ids=behavior_data.metadata.target_id.values,\n threshold_params=threshold\n )\n pred[action] = _pred * post\n del trainer\n gc.collect()\n else:\n continue\n del X_te\n gc.collect()\n\n if pred.shape[1] != 0:\n submission_part = predict_multiclass(pred, behavior_data.metadata)\n submission_list.append(submission_part)\n \n except KeyError:\n del behavior_data\n gc.collect()\n \n \n return submission_list","metadata":{"_uuid":"6743d14b-bd30-430d-8fe2-a84be56e18ce","_cell_guid":"25749e85-06cf-4739-90ad-14aaaeb8dbb2","trusted":true,"collapsed":false,"papermill":{"duration":0.018785,"end_time":"2025-10-30T12:15:01.450744","exception":false,"start_time":"2025-10-30T12:15:01.431959","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:52.684867Z","iopub.execute_input":"2025-12-14T16:49:52.685665Z","iopub.status.idle":"2025-12-14T16:49:52.709226Z","shell.execute_reply.started":"2025-12-14T16:49:52.685629Z","shell.execute_reply":"2025-12-14T16:49:52.708222Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thresholds = {\"single\": {}, \"pair\": {}}\n# thresholds = joblib.load(\"/kaggle/input/mabe-xgb-v2/xgboost/thresholds.pkl\")","metadata":{"_uuid":"a6ad5a44-5de0-41ef-a421-0780e4b2f5fb","_cell_guid":"f84b1717-90f8-43bc-9d06-b2ab9b372ee0","trusted":true,"collapsed":false,"execution":{"iopub.status.busy":"2025-12-14T16:49:52.710324Z","iopub.execute_input":"2025-12-14T16:49:52.71069Z","iopub.status.idle":"2025-12-14T16:49:52.730915Z","shell.execute_reply.started":"2025-12-14T16:49:52.710668Z","shell.execute_reply":"2025-12-14T16:49:52.729817Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"f1_list = []\nsubmission_list = []\n\nfor section in range(0, len(lab_list)):\n lab_id = lab_list[section]\n print(f\"{section}/{len(lab_list)-1} Processing videos with: {lab_id}\\n\")\n train_subset = train[train.lab_id == lab_id]\n\n if len(train_subset) == 0:\n print(f\"{lab_id} can`t be prediction\")\n continue\n\n train_preprocessor = Preprocessor(data_loader, mode='train', dataset=train_subset)\n print(train_preprocessor.body_parts_tracked)\n \n single_mouse_list = []\n mouse_pair_list = []\n try:\n for behavior_data in tqdm(train_preprocessor.generate_mouse_data()):\n if behavior_data.behavior_type == 'single':\n single_mouse_list.append(behavior_data)\n else:\n mouse_pair_list.append(behavior_data)\n gc.collect()\n except UnpredictionException:\n print(f\"{lab_id} can`t be prediction\")\n continue\n\n \n if len(single_mouse_list) > 0:\n if CFG.mode == 'validate':\n processed_single = generate_features_and_concat(single_mouse_list)\n del single_mouse_list\n \n # 学習\n temp_submission_list, temp_f1_list, temp_thresholds = cross_validate_classifier(processed_single, lab_id, section)\n \n if f\"{section}\" not in thresholds[\"single\"].keys():\n thresholds[\"single\"][f\"{section}\"] = {}\n for k, v in temp_thresholds.items():\n thresholds[\"single\"][f\"{section}\"][k] = v \n \n f1_list.extend(temp_f1_list)\n submission_list.extend(temp_submission_list)\n \n del temp_submission_list, temp_f1_list, temp_thresholds\n \n del processed_single\n gc.collect()\n else:\n del single_mouse_list\n gc.collect()\n temp_submission_list = submit(train_preprocessor.body_parts_tracked, 'single', section, lab_id)\n submission_list.extend(temp_submission_list)\n \n del temp_submission_list\n gc.collect()\n \n if len(mouse_pair_list) > 0:\n if CFG.mode == 'validate':\n processed_pair = generate_features_and_concat(mouse_pair_list)\n del mouse_pair_list\n gc.collect()\n \n # 学習\n temp_submission_list, temp_f1_list, temp_thresholds = cross_validate_classifier(processed_pair, lab_id, section)\n \n if f\"{section}\" not in thresholds[\"pair\"].keys():\n thresholds[\"pair\"][f\"{section}\"] = {}\n for k, v in temp_thresholds.items():\n thresholds[\"pair\"][f\"{section}\"][k] = v \n \n f1_list.extend(temp_f1_list)\n submission_list.extend(temp_submission_list)\n \n del temp_submission_list, temp_f1_list, temp_thresholds\n\n del processed_pair\n gc.collect()\n else:\n # 推論\n del mouse_pair_list\n gc.collect()\n temp_submission_list = submit(train_preprocessor.body_parts_tracked, 'pair', section, lab_id)\n \n submission_list.extend(temp_submission_list)\n del temp_submission_list\n gc.collect()\n \n print()","metadata":{"_uuid":"32402f97-5817-4e26-99e8-46e3f85a4cc1","_cell_guid":"f4b263e3-bcee-4e5b-b091-ddf52b51df12","trusted":true,"collapsed":false,"papermill":{"duration":1374.741754,"end_time":"2025-10-30T12:37:56.226485","exception":false,"start_time":"2025-10-30T12:15:01.484731","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2025-12-14T16:49:52.733292Z","iopub.execute_input":"2025-12-14T16:49:52.734138Z"},"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_uuid":"4d15fde9-ea1d-407d-9fce-60fef8828f6d","_cell_guid":"03e8ab76-ce22-4cf1-83ed-8b5acc7565f4","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"if CFG.mode == 'validate': \n submission = pd.concat(submission_list)\n submission_robust = robustify(submission, train, 'train') \n print(f\"Competition metric: {score(solution, submission_robust, ''):.4f}\")\n\n f1_df = pd.DataFrame(f1_list, columns=['body_parts_tracked_str', 'action', 'binary F1 score'])\n print(f\"Mean F1: {f1_df['binary F1 score'].mean():.4f}\")\n \n joblib.dump(thresholds, f\"{CFG.model_name}/thresholds.pkl\")\n joblib.dump(f1_df, f\"{CFG.model_name}/scores.pkl\")","metadata":{"_uuid":"dfd77bdf-a40b-4655-b729-6102c72865d4","_cell_guid":"4ace8702-9f26-4c7a-aacf-0ff9f518ab07","trusted":true,"collapsed":false,"papermill":{"duration":0.015287,"end_time":"2025-10-30T12:37:56.249092","exception":false,"start_time":"2025-10-30T12:37:56.233805","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_uuid":"c7ea05c4-576d-4028-b7f9-2f67d52f415e","_cell_guid":"c9458f1f-6ca5-4504-aff4-19de01011e79","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"markdown","source":"# Submission","metadata":{"_uuid":"009e72a6-647a-4d10-adbe-ea1d5c6533b3","_cell_guid":"1cebf8c3-41dd-4f6f-af3f-d38f800dda56","trusted":true,"collapsed":false,"papermill":{"duration":0.006717,"end_time":"2025-10-30T12:37:56.262811","exception":false,"start_time":"2025-10-30T12:37:56.256094","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"if CFG.mode == 'submit':\n submission = pd.concat(submission_list)\n submission_robust = robustify(submission, test, 'test')\n submission_robust.index.name = 'row_id'\n submission_robust.to_csv('submission.csv')\n submission.head()","metadata":{"_uuid":"c093118d-b63b-4fe1-96b5-af29d6c228c2","_cell_guid":"ed16b3ba-25ea-4960-9b31-23b253270a9f","trusted":true,"collapsed":false,"papermill":{"duration":0.118309,"end_time":"2025-10-30T12:37:56.387939","exception":false,"start_time":"2025-10-30T12:37:56.26963","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"","metadata":{"_uuid":"012de1d7-6eb7-497e-a834-ac9280cfeadd","_cell_guid":"d5f3387b-c89c-49ea-89c7-4f62444d8800","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"def plot_timeline_with_evaluation(gt, pred, video_id, agent_id, target_id):\n \"\"\"\n 特定のvideo_id, agent_id, target_idに対してタイムラインと正誤判定を可視化\n \"\"\"\n gt_subset = gt[(gt['video_id'] == video_id) & \n (gt['agent_id'] == agent_id) & \n (gt['target_id'] == target_id)].copy()\n pred_subset = pred[(pred['video_id'] == video_id) & \n (pred['agent_id'] == agent_id) & \n (pred['target_id'] == target_id)].copy()\n \n # 全てのユニークなactionを取得してカラーマップを作成\n all_actions = pd.concat([gt['action'], pred['action']]).unique()\n colors = plt.cm.tab10(range(len(all_actions)))\n action_colors = dict(zip(all_actions, colors))\n \n fig, ax = plt.subplots(figsize=(16, 5))\n \n # Ground Truthをプロット\n for idx, row in gt_subset.iterrows():\n ax.barh(1, row['stop_frame'] - row['start_frame'], \n left=row['start_frame'], height=0.4, \n color=action_colors[row['action']], \n edgecolor='black', linewidth=1.5, alpha=0.8)\n mid_point = (row['start_frame'] + row['stop_frame']) / 2\n ax.text(mid_point, 1, row['action'], \n ha='center', va='center', fontsize=9, fontweight='bold')\n \n # Predictionをプロット\n for idx, row in pred_subset.iterrows():\n ax.barh(2, row['stop_frame'] - row['start_frame'], \n left=row['start_frame'], height=0.4, \n color=action_colors[row['action']], \n edgecolor='black', linewidth=1.5, alpha=0.8)\n mid_point = (row['start_frame'] + row['stop_frame']) / 2\n ax.text(mid_point, 2, row['action'], \n ha='center', va='center', fontsize=9, fontweight='bold')\n \n # 正誤判定の可視化\n # 全フレームの範囲を取得\n all_frames = pd.concat([\n gt_subset[['start_frame', 'stop_frame']], \n pred_subset[['start_frame', 'stop_frame']]\n ])\n min_frame = int(all_frames.min().min())\n max_frame = int(all_frames.max().max())\n \n # フレームごとにGTとPredのactionを記録\n gt_actions = {}\n pred_actions = {}\n \n for _, row in gt_subset.iterrows():\n for frame in range(int(row['start_frame']), int(row['stop_frame'])):\n gt_actions[frame] = row['action']\n \n for _, row in pred_subset.iterrows():\n for frame in range(int(row['start_frame']), int(row['stop_frame'])):\n pred_actions[frame] = row['action']\n \n # 連続する同じ状態のフレームをまとめて描画\n current_start = None\n current_status = None\n \n for frame in range(min_frame, max_frame + 1):\n gt_action = gt_actions.get(frame, None)\n pred_action = pred_actions.get(frame, None)\n \n # TP, FP, FN, TNの判定\n if gt_action is not None and pred_action is not None:\n if gt_action == pred_action:\n status = 'TP' # True Positive（正しく予測）\n else:\n status = 'FP' # False Positive（誤った予測）\n elif gt_action is None and pred_action is not None:\n status = 'FP' # False Positive\n elif gt_action is not None and pred_action is None:\n status = 'FN' # False Negative（予測漏れ）\n else:\n status = 'TN' # True Negative（何もない）\n \n # 状態が変わったら前のセグメントを描画\n if current_status is None:\n current_start = frame\n current_status = status\n elif current_status != status:\n # 前のセグメントを描画\n if current_status == 'TP':\n color = 'blue'\n alpha = 0.6\n elif current_status in ['FP', 'FN']:\n color = 'red'\n alpha = 0.6\n else: # TN\n color = 'lightgray'\n alpha = 0.3\n \n ax.barh(0, frame - current_start, \n left=current_start, height=0.4, \n color=color, edgecolor='none', alpha=alpha)\n \n current_start = frame\n current_status = status\n \n # 最後のセグメントを描画\n if current_status is not None:\n if current_status == 'TP':\n color = 'blue'\n alpha = 0.6\n elif current_status in ['FP', 'FN']:\n color = 'red'\n alpha = 0.6\n else: # TN\n color = 'lightgray'\n alpha = 0.3\n \n ax.barh(0, max_frame - current_start, \n left=current_start, height=0.4, \n color=color, edgecolor='none', alpha=alpha)\n \n ax.set_yticks([0, 1, 2])\n ax.set_yticklabels(['Evaluation\\n(TP/FP/FN)', 'Ground Truth', 'Prediction'], fontsize=11)\n ax.set_xlabel('Frame', fontsize=12)\n ax.set_title(f'Video {video_id}, Agent {agent_id}, Target {target_id}', \n fontsize=14, fontweight='bold')\n ax.grid(True, axis='x', alpha=0.3)\n \n # 凡例を作成（actionの色）\n legend_patches = [mpatches.Patch(color=action_colors[action], label=action) \n for action in all_actions]\n # 評価指標の凡例を追加\n legend_patches.extend([\n mpatches.Patch(color='blue', alpha=0.6, label='TP (Correct)'),\n mpatches.Patch(color='red', alpha=0.6, label='FP/FN (Incorrect)')\n ])\n \n # ax.legend(handles=legend_patches, loc='upper right', \n # bbox_to_anchor=(1.15, 1), fontsize=10)\n \n plt.tight_layout()\n plt.show()\n\n\ndef plot_all_timelines_with_evaluation(gt, pred):\n \"\"\"\n 全てのvideo_id, agent_id, target_idの組み合わせに対してタイムラインを可視化\n \"\"\"\n gt_combinations = gt[['video_id', 'agent_id', 'target_id']].drop_duplicates()\n \n for idx, row in gt_combinations.iterrows():\n plot_timeline_with_evaluation(gt, pred, row['video_id'], row['agent_id'], row['target_id'])","metadata":{"_uuid":"bc8ff407-a004-46ee-b76e-87ec2821935d","_cell_guid":"3caa577d-cf2b-4543-9788-99a178dd5af8","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# for v_id in solution[\"video_id\"].unique():\n# __s = solution[solution[\"video_id\"] == v_id]\n# for a, t in __s[[\"agent_id\", \"target_id\"]].drop_duplicates().values:\n# plot_timeline_with_evaluation(solution, submission, v_id, a, t)","metadata":{"_uuid":"e8341e81-7854-45cd-9a74-f33c67496fc6","_cell_guid":"2e0a387a-8a7d-4986-90e4-fd159b3e10f9","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"c3fb0966-b2b7-4b7f-82dd-06b0df47e7e4","_cell_guid":"bf1171f6-693f-4643-884a-cffd99aae37e","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"2d979184-4f76-4d5e-8156-2a71f246196d","_cell_guid":"bc501328-051e-4c55-96f5-adaaffcfa0d3","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"e00f8eb7-c89f-4529-88a9-70d16e7f8a25","_cell_guid":"9bbc2b87-bd61-43ad-84df-21016ddc2faa","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"7186a250-9c57-4db9-9616-b74f79a8f8b3","_cell_guid":"3f80b743-4204-455f-8172-21411de9e80a","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"b03b94f7-e24c-4292-963d-d1ee764034e3","_cell_guid":"4d41e8af-0380-4349-8e95-bfac570936b2","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"_uuid":"31716f8a-5e37-4aa1-b99e-5ede4b1585e0","_cell_guid":"77295f60-ab87-4a0f-b509-2a3522fe9324","trusted":true,"collapsed":false,"jupyter":{"outputs_hidden":false}},"outputs":[],"execution_count":null}]}