{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":14127776,"sourceType":"datasetVersion","datasetId":8684113},{"sourceId":273367188,"sourceType":"kernelVersion"},{"sourceId":284978406,"sourceType":"kernelVersion"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":1494.813501,"end_time":"2025-10-25T19:51:35.870094","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-25T19:26:41.056593","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00962fe9010e42b7a5b123af50d2a026":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b098fa23a3b4031bf88b32b20044978":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"108c3e5db1154f47a5aca68386ed4f24":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"38c36db3be3843b19da108d4cd1f9421":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e9806911d1d489d963f76955d8468c4","IPY_MODEL_e6ad60bc760e4b51885b15335c048a65","IPY_MODEL_6f4723180bf14101b60deeac9fd59e50"],"layout":"IPY_MODEL_00962fe9010e42b7a5b123af50d2a026","tabbable":null,"tooltip":null}},"3e9806911d1d489d963f76955d8468c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_aeb2544f9ad9413b8b71b5f5643b1019","placeholder":"​","style":"IPY_MODEL_ec012e9ac53d477daee0988a35117f0b","tabbable":null,"tooltip":null,"value":"100%"}},"3edd91e1eafb4f6bbf55275ea60b6c97":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f4723180bf14101b60deeac9fd59e50":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e8f95c8096294ce6b6fb2b65c08af536","placeholder":"​","style":"IPY_MODEL_0b098fa23a3b4031bf88b32b20044978","tabbable":null,"tooltip":null,"value":" 863/863 [00:08&lt;00:00, 96.46it/s]"}},"aeb2544f9ad9413b8b71b5f5643b1019":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e6ad60bc760e4b51885b15335c048a65":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_3edd91e1eafb4f6bbf55275ea60b6c97","max":863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_108c3e5db1154f47a5aca68386ed4f24","tabbable":null,"tooltip":null,"value":863}},"e8f95c8096294ce6b6fb2b65c08af536":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec012e9ac53d477daee0988a35117f0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.007171,"end_time":"2025-10-25T19:26:46.426493","exception":false,"start_time":"2025-10-25T19:26:46.419322","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# !pip install xgboost==3.1.1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:29.053297Z","iopub.execute_input":"2025-12-12T14:43:29.053581Z","iopub.status.idle":"2025-12-12T14:43:33.013552Z","shell.execute_reply.started":"2025-12-12T14:43:29.053563Z","shell.execute_reply":"2025-12-12T14:43:33.012109Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ls /kaggle/input/mabe2025-models/","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.015385Z","iopub.execute_input":"2025-12-12T14:43:33.015722Z","iopub.status.idle":"2025-12-12T14:43:33.161436Z","shell.execute_reply.started":"2025-12-12T14:43:33.015691Z","shell.execute_reply":"2025-12-12T14:43:33.160135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.base import clone\nimport xgboost\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nfrom lightgbm import LGBMClassifier\nfrom tqdm.notebook import tqdm\nimport pandas as pd\nfrom typing import Tuple\nimport json\n# from koolbox import Trainer\nimport numpy as np\nimport itertools\nimport warnings\nimport optuna\nimport joblib\nimport glob\nimport gc\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')\n\nxgboost.__version__","metadata":{"papermill":{"duration":8.578306,"end_time":"2025-10-25T19:26:57.993688","exception":false,"start_time":"2025-10-25T19:26:49.415382","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.163584Z","iopub.execute_input":"2025-12-12T14:43:33.163978Z","iopub.status.idle":"2025-12-12T14:43:33.174256Z","shell.execute_reply.started":"2025-12-12T14:43:33.163934Z","shell.execute_reply":"2025-12-12T14:43:33.173134Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ls /kaggle/input/mabe2025-models/9f2cadf17/9f2cadf17","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.176432Z","iopub.execute_input":"2025-12-12T14:43:33.176682Z","iopub.status.idle":"2025-12-12T14:43:33.193204Z","shell.execute_reply.started":"2025-12-12T14:43:33.176664Z","shell.execute_reply":"2025-12-12T14:43:33.192176Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.00762,"end_time":"2025-10-25T19:26:58.057724","exception":false,"start_time":"2025-10-25T19:26:58.050104","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n\n    datapath = \"/kaggle/input/MABe-mouse-behavior-detection\"\n    train_path = \"/kaggle/input/MABe-mouse-behavior-detection/train.csv\"\n    test_path = \"/kaggle/input/MABe-mouse-behavior-detection/test.csv\"\n    train_annotation_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n    train_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n    test_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\"\n\n    # modèle unique\n    # model_base_path = \"cc52d18ef\"\n\n    # modèle par section\n    model_base_path = {\n        1: \"5d66c0818\",\n        2: \"5d66c0818\",\n        3: \"5d66c0818\",\n        4: \"5d66c0818\",\n        5: \"5d66c0818\",\n        6: \"5d66c0818\",\n        7: \"fd901105e\",\n        8: \"fd901105e\",\n        9: \"fd901105e\",\n    }\n\n    # # modèle par section\n    # model_base_path = {\n    #     1: \"fee9e7955\",\n    #     2: \"fee9e7955\",\n    #     3: \"c95b890c5\",\n    #     4: \"c95b890c5\",\n    #     5: \"fee9e7955\",\n    #     6: \"fee9e7955\",\n    #     7: \"c95b890c5\",\n    #     8: \"c95b890c5\",\n    #     9: \"c95b890c5\",\n    # }\n    \n    \n    mode = \"submit\"\n    \n    debug_predict = False\n    ","metadata":{"papermill":{"duration":0.014723,"end_time":"2025-10-25T19:26:58.043155","exception":false,"start_time":"2025-10-25T19:26:58.028432","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.194338Z","iopub.execute_input":"2025-12-12T14:43:33.194573Z","iopub.status.idle":"2025-12-12T14:43:33.216012Z","shell.execute_reply.started":"2025-12-12T14:43:33.194555Z","shell.execute_reply":"2025-12-12T14:43:33.214796Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv(CFG.train_path)\ntrain['n_mice'] = 4 - train[['mouse1_strain', 'mouse2_strain', 'mouse3_strain', 'mouse4_strain']].isna().sum(axis=1)\ntrain_without_mabe22 = train.query(\"~lab_id.str.startswith('MABe22_')\")\n\ntest = pd.read_csv(CFG.test_path)\n\nbody_parts_tracked_list = list(np.unique(train.body_parts_tracked))\n\nif CFG.debug_predict:\n    # test = train\n    test = train_without_mabe22.groupby(\"lab_id\").head(3).reset_index(drop=True)\n    print(f\"running eval on train: {test.shape[0]} rows\")","metadata":{"papermill":{"duration":0.197857,"end_time":"2025-10-25T19:26:58.262128","exception":false,"start_time":"2025-10-25T19:26:58.064271","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.217190Z","iopub.execute_input":"2025-12-12T14:43:33.217509Z","iopub.status.idle":"2025-12-12T14:43:33.354513Z","shell.execute_reply.started":"2025-12-12T14:43:33.217475Z","shell.execute_reply":"2025-12-12T14:43:33.353434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data generator","metadata":{"papermill":{"duration":0.006479,"end_time":"2025-10-25T19:26:58.343672","exception":false,"start_time":"2025-10-25T19:26:58.337193","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_body_parts =  [\n    'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n    'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright', \n    'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint'\n]\n\n\nLAB_MAPPING = {\n    \"AdaptableSnail\": 0,\n    \"BoisterousParrot\": 1,\n    \"CRIM13\": 2,\n    \"CalMS21_supplemental\": 3,\n    \"CalMS21_task1\": 4,\n    \"CalMS21_task2\": 5,\n    \"CautiousGiraffe\": 6,\n    \"DeliriousFly\": 7,\n    \"ElegantMink\": 8,\n    \"GroovyShrew\": 9,\n    \"InvincibleJellyfish\": 10,\n    \"JovialSwallow\": 11,\n    \"LyricalHare\": 12,\n    \"MABe22_keypoints\": 13,\n    \"MABe22_movies\": 14,\n    \"NiftyGoldfinch\": 15,\n    \"PleasantMeerkat\": 16,\n    \"ReflectiveManatee\": 17,\n    \"SparklingTapir\": 18,\n    \"TranquilPanther\": 19,\n    \"UppityFerret\": 20,\n}\n\ndef generate_mouse_data(\n    CFG,\n    dataset,\n    drop_body_parts,\n    traintest,\n    traintest_directory=None,\n    generate_single=True,\n    generate_pair=True,\n):\n    if traintest_directory is None:\n        traintest_directory = f\"{CFG.datapath}/{traintest}_tracking\"\n\n    for _, row in tqdm(\n        dataset.iterrows(), total=len(dataset), desc=f\"Generating {traintest} data\"\n    ):\n        lab_id = row.lab_id\n        if lab_id.startswith(\"MABe22\") or not isinstance(row.behaviors_labeled, str):\n            continue\n\n        video_id = row.video_id\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n        if len(np.unique(vid.bodypart)) > 5:\n            vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n\n        pvid = vid.pivot(\n            columns=[\"mouse_id\", \"bodypart\"], index=\"video_frame\", values=[\"x\", \"y\"]\n        )\n\n        del vid\n        gc.collect()\n\n        pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n        pvid /= row.pix_per_cm_approx\n\n        vid_behaviors = json.loads(row.behaviors_labeled)\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(\",\") for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(\n            vid_behaviors, columns=[\"agent\", \"target\", \"action\"]\n        )\n\n        # #####################################\n        # Example of 'vid_behaviors' structure:\n        # ######################################\n        #    agent   target     action\n        # 0  mouse1  mouse2     attack\n        # 1  mouse1  mouse2  dominance\n        # 2  mouse1  mouse2      sniff\n        if traintest == \"train\":\n            try:\n                annot = pd.read_parquet(\n                    path.replace(\"train_tracking\", \"train_annotation\")\n                )\n            except FileNotFoundError:\n                continue\n\n        # ===============================================\n        # Generate single mouse data, ie \"target == self\"\n        # ===============================================\n        if generate_single:\n            vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n            for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n                try:\n                    mouse_id = int(mouse_id_str[-1])\n                    vid_agent_actions = np.unique(\n                        vid_behaviors_subset.query(\"agent == @mouse_id_str\").action\n                    )\n                    single_mouse = pvid.loc[:, mouse_id]\n                    assert len(single_mouse) == len(pvid)\n                    single_mouse_meta = pd.DataFrame(\n                        {\n                            \"video_id\": video_id,\n                            \"agent_id\": mouse_id_str,\n                            \"target_id\": \"self\",\n                            \"video_frame\": single_mouse.index,\n                            \"sex\": row[f\"{mouse_id_str}_sex\"],\n                            \"color\": row[f\"{mouse_id_str}_color\"],\n                            \"age\": row[f\"{mouse_id_str}_age\"],\n                            \"lab_id\": LAB_MAPPING[lab_id],\n                             \"arena_width_pixels\": (\n                                row[\"arena_width_cm\"] * row[\"pix_per_cm_approx\"]\n                            ),\n                            \"arena_height_pixels\": (\n                                row[\"arena_height_cm\"] * row[\"pix_per_cm_approx\"]\n                            ),\n                        }\n                    )\n                    if traintest == \"train\":\n                        single_mouse_label = pd.DataFrame(\n                            0.0, columns=vid_agent_actions, index=single_mouse.index\n                        )\n                        annot_subset = annot.query(\n                            \"(agent_id == @mouse_id) & (target_id == @mouse_id)\"\n                        )\n                        for i in range(len(annot_subset)):\n                            annot_row = annot_subset.iloc[i]\n                            single_mouse_label.loc[\n                                annot_row[\"start_frame\"] : annot_row[\"stop_frame\"],\n                                annot_row.action,\n                            ] = 1.0\n                        yield (\n                            \"single\",\n                            single_mouse,\n                            single_mouse_meta,\n                            single_mouse_label,\n                        )\n                    else:\n                        yield (\n                            \"single\",\n                            single_mouse,\n                            single_mouse_meta,\n                            vid_agent_actions,\n                        )\n                except KeyError:\n                    pass\n\n        # ========================================================\n        # Generate mouse interactions data, ie \"target != self\"\n        # ========================================================\n        if generate_pair:\n            vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n            if len(vid_behaviors_subset) > 0:\n                for agent, target in itertools.permutations(\n                    np.unique(pvid.columns.get_level_values(\"mouse_id\")), 2\n                ):  # int8\n                    agent_str = f\"mouse{agent}\"\n                    target_str = f\"mouse{target}\"\n                    vid_agent_actions = np.unique(\n                        vid_behaviors_subset.query(\n                            \"(agent == @agent_str) & (target == @target_str)\"\n                        ).action\n                    )\n                    mouse_pair = pd.concat(\n                        [pvid[agent], pvid[target]], axis=1, keys=[\"A\", \"B\"]\n                    )\n                    assert len(mouse_pair) == len(pvid)\n                    mouse_pair_meta = pd.DataFrame(\n                        {\n                            \"video_id\": video_id,\n                            \"agent_id\": agent_str,\n                            \"target_id\": target_str,\n                            \"video_frame\": mouse_pair.index,\n                            \"agent_sex\": row[f\"{agent_str}_sex\"],\n                            \"agent_color\": row[f\"{agent_str}_color\"],\n                            \"agent_age\": row[f\"{agent_str}_age\"],\n                            \"target_sex\": row[f\"{target_str}_sex\"],\n                            \"target_color\": row[f\"{target_str}_color\"],\n                            \"target_age\": row[f\"{target_str}_age\"],\n                            \"lab_id\": LAB_MAPPING[lab_id],\n                            \"arena_width_pixels\": (\n                                row[\"arena_width_cm\"] * row[\"pix_per_cm_approx\"]\n                            ),\n                            \"arena_height_pixels\": (\n                                row[\"arena_height_cm\"] * row[\"pix_per_cm_approx\"]\n                            ),\n                        }\n                    )\n                    # print(mouse_pair_meta)\n                    # breakpoint()\n\n                    if traintest == \"train\":\n                        mouse_pair_label = pd.DataFrame(\n                            0.0, columns=vid_agent_actions, index=mouse_pair.index\n                        )\n                        annot_subset = annot.query(\n                            \"(agent_id == @agent) & (target_id == @target)\"\n                        )\n                        for i in range(len(annot_subset)):\n                            annot_row = annot_subset.iloc[i]\n                            mouse_pair_label.loc[\n                                annot_row[\"start_frame\"] : annot_row[\"stop_frame\"],\n                                annot_row.action,\n                            ] = 1.0\n                        yield \"pair\", mouse_pair, mouse_pair_meta, mouse_pair_label\n                    else:\n                        yield \"pair\", mouse_pair, mouse_pair_meta, vid_agent_actions","metadata":{"papermill":{"duration":0.028134,"end_time":"2025-10-25T19:26:58.378526","exception":false,"start_time":"2025-10-25T19:26:58.350392","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.355983Z","iopub.execute_input":"2025-12-12T14:43:33.356310Z","iopub.status.idle":"2025-12-12T14:43:33.378200Z","shell.execute_reply.started":"2025-12-12T14:43:33.356285Z","shell.execute_reply":"2025-12-12T14:43:33.377106Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transforming coordinates","metadata":{"papermill":{"duration":0.006144,"end_time":"2025-10-25T19:26:58.391332","exception":false,"start_time":"2025-10-25T19:26:58.385188","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def _get_sex(meta, column=\"sex\"):\n    sex = meta[column].unique()[0]\n    if pd.isna(sex):\n        return np.nan\n    return 0 if sex == \"male\" else 1\n    \ndef safe_rolling(series, window, func, min_periods=None):\n    if min_periods is None:\n        min_periods = max(1, window // 4)\n    return series.rolling(window, min_periods=min_periods, center=True).apply(func, raw=True)\n\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n    if n_frames_at_30fps == 0:\n        return 0\n    s = 1 if n_frames_at_30fps > 0 else -1\n    mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n    return s * mag\n\n\n\ndef add_velocity_features(X, cx, cy, fps):\n    vx = cx.diff() * fps\n    vy = cy.diff() * fps\n    X[\"vel_x\"] = vx\n    X[\"vel_y\"] = vy\n    X[\"vel_mag\"] = np.sqrt(vx**2 + vy**2)\n    return X\n\n\ndef add_acceleration_features(X, cx, cy, fps):\n    ax = cx.diff().diff() * fps**2\n    ay = cy.diff().diff() * fps**2\n    X[\"acc_x\"] = ax\n    X[\"acc_y\"] = ay\n    X[\"acc_mag\"] = np.sqrt(ax**2 + ay**2)\n    return X\n\n\ndef add_heading_features(X, cx, cy):\n    vx = cx.diff()\n    vy = cy.diff()\n    heading = np.arctan2(vy, vx)\n    X[\"heading\"] = heading\n    X[\"heading_change\"] = heading.diff().abs()\n    return X\n\n\ndef add_stationarity_features(X, cx, cy, fps):\n    speed = np.sqrt(cx.diff() ** 2 + cy.diff() ** 2) * fps\n    X[\"is_still\"] = (speed < 1.0).astype(float)\n    X[\"still_run\"] = X[\"is_still\"].rolling(30, min_periods=1).sum()\n    return X\n\n\ndef add_pathline_features(X, cx, cy, fps):\n    displacement = np.sqrt(cx.diff() ** 2 + cy.diff() ** 2)\n    X[\"path_cumul\"] = displacement.cumsum()\n    for w in [30, 60, 120]:\n        ws = _scale(w, fps)\n        X[f\"path_local_{w}\"] = displacement.rolling(ws, min_periods=1).sum()\n    return X\n\n\ndef fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n    if 'frames_per_second' in meta_df.columns and pd.notnull(meta_df['frames_per_second']).any():\n        return float(meta_df['frames_per_second'].iloc[0])\n    vid = meta_df['video_id'].iloc[0]\n    return float(fallback_lookup.get(vid, default_fps))\n\ndef add_curvature_features(X, center_x, center_y, fps):\n    vel_x = center_x.diff()\n    vel_y = center_y.diff()\n    acc_x = vel_x.diff()\n    acc_y = vel_y.diff()\n\n    cross_prod = vel_x * acc_y - vel_y * acc_x\n    vel_mag = np.sqrt(vel_x**2 + vel_y**2)\n    curvature = np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n\n    for w in [25, 50, 75]:\n        ws = _scale(w, fps)\n        X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 5)).mean()\n\n    angle = np.arctan2(vel_y, vel_x)\n    angle_change = np.abs(angle.diff())\n    w = 30\n    ws = _scale(w, fps)\n    X[f'turn_rate_{w}'] = angle_change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n\n    return X\n\ndef add_multiscale_features(X, center_x, center_y, fps):\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n\n    scales = [20, 40, 60, 80]\n    for scale in scales:\n        ws = _scale(scale, fps)\n        if len(speed) >= ws:\n            X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n            X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n\n    if len(scales) >= 2 and f'sp_m{scales[0]}' in X.columns and f'sp_m{scales[-1]}' in X.columns:\n        X['sp_ratio'] = X[f'sp_m{scales[0]}'] / (X[f'sp_m{scales[-1]}'] + 1e-6)\n\n    return X\n\ndef add_state_features(X, center_x, center_y, fps):\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n    w_ma = _scale(15, fps)\n    speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n\n    try:\n        bins = [-np.inf, 0.5 * fps, 2.0 * fps, 5.0 * fps, np.inf]\n        speed_states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n\n        for window in [20, 40, 60, 80]:\n            ws = _scale(window, fps)\n            if len(speed_states) >= ws:\n                for state in [0, 1, 2, 3]:\n                    X[f's{state}_{window}'] = (\n                        (speed_states == state).astype(float)\n                        .rolling(ws, min_periods=max(1, ws // 5)).mean()\n                    )\n                state_changes = (speed_states != speed_states.shift(1)).astype(float)\n                X[f'trans_{window}'] = state_changes.rolling(ws, min_periods=max(1, ws // 5)).sum()\n    except Exception:\n        pass\n\n    return X\n\ndef add_longrange_features(X, center_x, center_y, fps):\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(center_x) >= ws:\n            X[f'x_ml{window}'] = center_x.rolling(ws, min_periods=max(5, ws // 6)).mean()\n            X[f'y_ml{window}'] = center_y.rolling(ws, min_periods=max(5, ws // 6)).mean()\n\n    for span in [30, 60, 120]:\n        s = _scale(span, fps)\n        X[f'x_e{span}'] = center_x.ewm(span=s, min_periods=1).mean()\n        X[f'y_e{span}'] = center_y.ewm(span=s, min_periods=1).mean()\n\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)  # cm/s\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(speed) >= ws:\n            X[f'sp_pct{window}'] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n\n    return X\n\ndef add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n    if 'body_center' not in avail_A or 'body_center' not in avail_B:\n        return X\n\n    rel_x = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n    rel_y = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n    rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n    A_vx = mouse_pair['A']['body_center']['x'].diff()\n    A_vy = mouse_pair['A']['body_center']['y'].diff()\n    B_vx = mouse_pair['B']['body_center']['x'].diff()\n    B_vy = mouse_pair['B']['body_center']['y'].diff()\n\n    A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n    B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n    for window in [30, 60, 90, 120]:\n        ws = _scale(window, fps)\n        X[f'A_ld{window}'] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n        X[f'B_ld{window}'] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    approach = -rel_dist.diff()\n    chase = approach * B_lead\n    w = 30\n    ws = _scale(w, fps)\n    X[f'chase_{w}'] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    for window in [60, 120]:\n        ws = _scale(window, fps)\n        A_sp = np.sqrt(A_vx**2 + A_vy**2)\n        B_sp = np.sqrt(B_vx**2 + B_vy**2)\n        X[f'sp_cor{window}'] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n\n    return X","metadata":{"papermill":{"duration":0.038766,"end_time":"2025-10-25T19:26:58.43742","exception":false,"start_time":"2025-10-25T19:26:58.398654","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.379459Z","iopub.execute_input":"2025-12-12T14:43:33.379966Z","iopub.status.idle":"2025-12-12T14:43:33.424395Z","shell.execute_reply.started":"2025-12-12T14:43:33.379933Z","shell.execute_reply":"2025-12-12T14:43:33.423076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transform_single(single_mouse, body_parts_tracked, fps, arena_size):\n    available_body_parts = single_mouse.columns.get_level_values(0)\n\n    # ==============================================================\n    # features n°1: distance squared between all body part pairs\n    # ==============================================================\n    X = pd.DataFrame(\n        {\n            f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(\n                axis=1, skipna=False\n            )\n            for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n            if p1 in available_body_parts and p2 in available_body_parts\n        }\n    )\n\n    X = X.reindex(\n        columns=[\n            f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n        ],\n        copy=False,\n    )\n\n    to_keep = [\"body_center\", \"nose\", \"tail_tip\", \"head\", \"tail_base\"]\n    norm_x = pd.concat(\n        [\n            single_mouse[bp][\"x\"] / arena_size[0]\n            for bp in body_parts_tracked\n            if bp in to_keep\n        ],\n        axis=1,\n    )\n    norm_x.columns = [f\"{bp}_x_norm\" for bp in body_parts_tracked if bp in to_keep]\n\n    norm_y = pd.concat(\n        [\n            single_mouse[bp][\"y\"] / arena_size[0]\n            for bp in body_parts_tracked\n            if bp in to_keep\n        ],\n        axis=1,\n    )\n    norm_y.columns = [f\"{bp}_y_norm\" for bp in body_parts_tracked if bp in to_keep]\n\n    normed_coords = norm_x.join(norm_y)\n\n    X = X.join(normed_coords)\n    \n    # ==============================================================\n    # features n°2: speeds of ears and tail_base\n    # ==============================================================\n    if all(p in single_mouse.columns for p in [\"ear_left\", \"ear_right\", \"tail_base\"]):\n        for lag_val in [5, 10, 15, 20]:\n            lag = _scale(lag_val, fps)  # number of frames to lag\n\n            shifted = single_mouse[[\"ear_left\", \"ear_right\", \"tail_base\"]].shift(lag)\n            speeds = pd.DataFrame(\n                {\n                    f\"sp_lf_lag{lag_val}\": np.square(\n                        single_mouse[\"ear_left\"] - shifted[\"ear_left\"]\n                    ).sum(axis=1, skipna=False),\n                    f\"sp_rt_lag{lag_val}\": np.square(\n                        single_mouse[\"ear_right\"] - shifted[\"ear_right\"]\n                    ).sum(axis=1, skipna=False),\n                    f\"sp_lf2_lag{lag_val}\": np.square(\n                        single_mouse[\"ear_left\"] - shifted[\"tail_base\"]\n                    ).sum(axis=1, skipna=False),\n                    f\"sp_rt2_{lag_val}\": np.square(\n                        single_mouse[\"ear_right\"] - shifted[\"tail_base\"]\n                    ).sum(axis=1, skipna=False),\n                }\n            )\n            X = pd.concat([X, speeds], axis=1)\n\n    # if all(p in single_mouse.columns for p in [\"neck\", \"nose\", \"tail_base\"]):\n    #     lag = _scale(10, fps)  # number of frames to lag\n\n    #     shifted = single_mouse[[\"neck\", \"nose\", \"tail_base\"]].shift(lag)\n    #     speeds = pd.DataFrame(\n    #         {\n    #             f\"neck_past{5}_tail_base_dist\": np.square(\n    #                 single_mouse[\"neck\"] - shifted[\"tail_base\"]\n    #             ).sum(axis=1, skipna=False),\n    #             f\"nose_past{5}_tail_base_dist\": np.square(\n    #                 single_mouse[\"nose\"] - shifted[\"tail_base\"]\n    #             ).sum(axis=1, skipna=False),\n    #         }\n    #     )\n    #     X = pd.concat([X, speeds], axis=1)\n\n    # features n°3: longueur / largeur approximative du corps\n    if \"nose+tail_base\" in X.columns and \"ear_left+ear_right\" in X.columns:\n        X[\"elong\"] = X[\"nose+tail_base\"] / (X[\"ear_left+ear_right\"] + 1e-6)\n\n    # features n°4:\n    if all(p in available_body_parts for p in [\"nose\", \"body_center\", \"tail_base\"]):\n        v1 = single_mouse[\"nose\"] - single_mouse[\"body_center\"]\n        v2 = single_mouse[\"tail_base\"] - single_mouse[\"body_center\"]\n        X[\"body_ang\"] = (v1[\"x\"] * v2[\"x\"] + v1[\"y\"] * v2[\"y\"]) / (\n            np.sqrt(v1[\"x\"] ** 2 + v1[\"y\"] ** 2) * np.sqrt(v2[\"x\"] ** 2 + v2[\"y\"] ** 2)\n            + 1e-6\n        )\n\n    # Features n°5\n    if \"body_center\" in available_body_parts:\n        cx = single_mouse[\"body_center\"][\"x\"]\n        cy = single_mouse[\"body_center\"][\"y\"]\n\n        # --- New features integration ----------------------------------------\n        X = add_velocity_features(X, cx, cy, fps)\n        X = add_acceleration_features(X, cx, cy, fps)\n        X = add_heading_features(X, cx, cy)\n        X = add_stationarity_features(X, cx, cy, fps)\n        X = add_pathline_features(X, cx, cy, fps)\n        # ----------------------------------------------------------------------\n        \n        for w in [5, 10, 15, 20, 30, 45, 60, 90]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"cx_m{w}\"] = cx.rolling(ws, **roll).mean()\n            X[f\"cy_m{w}\"] = cy.rolling(ws, **roll).mean()\n            X[f\"cx_s{w}\"] = cx.rolling(ws, **roll).std()\n            X[f\"cy_s{w}\"] = cy.rolling(ws, **roll).std()\n            X[f\"x_rng{w}\"] = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n            X[f\"y_rng{w}\"] = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n            X[f\"disp{w}\"] = np.sqrt(\n                cx.diff().rolling(ws, min_periods=1).sum() ** 2\n                + cy.diff().rolling(ws, min_periods=1).sum() ** 2\n            )\n            X[f\"act{w}\"] = np.sqrt(\n                cx.diff().rolling(ws, min_periods=1).var()\n                + cy.diff().rolling(ws, min_periods=1).var()\n            )\n\n        X = add_curvature_features(X, cx, cy, fps)\n        X = add_multiscale_features(X, cx, cy, fps)\n        X = add_state_features(X, cx, cy, fps)\n        X = add_longrange_features(X, cx, cy, fps)\n\n    # Features n°6\n    if all(p in available_body_parts for p in [\"nose\", \"tail_base\"]):\n        nt_dist = np.sqrt(\n            (single_mouse[\"nose\"][\"x\"] - single_mouse[\"tail_base\"][\"x\"]) ** 2\n            + (single_mouse[\"nose\"][\"y\"] - single_mouse[\"tail_base\"][\"y\"]) ** 2\n        )\n        for lag in [10, 20, 30, 40, 50]:\n            l = _scale(lag, fps)\n            X[f\"nt_lg{lag}\"] = nt_dist.shift(l)\n            X[f\"nt_df{lag}\"] = nt_dist - nt_dist.shift(l)\n\n    # Features n°7\n    if all(p in available_body_parts for p in [\"ear_left\", \"ear_right\"]):\n        ear_d = np.sqrt(\n            (single_mouse[\"ear_left\"][\"x\"] - single_mouse[\"ear_right\"][\"x\"]) ** 2\n            + (single_mouse[\"ear_left\"][\"y\"] - single_mouse[\"ear_right\"][\"y\"]) ** 2\n        )\n        for off in [-30, -20, -10, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f\"ear_o{off}\"] = ear_d.shift(-o)\n        w = _scale(30, fps)\n        X[\"ear_con\"] = ear_d.rolling(w, min_periods=1, center=True).std() / (\n            ear_d.rolling(w, min_periods=1, center=True).mean() + 1e-6\n        )\n\n    return X.astype(np.float32, copy=False)\n\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps, arena_size):\n    avail_A = mouse_pair[\"A\"].columns.get_level_values(0)  # liste des bodyparts pour A\n    avail_B = mouse_pair[\"B\"].columns.get_level_values(0)  # liste des bodyparts pour B\n\n    # ==============================================================\n    # features n°1: distance squared between all body part pairs\n    # ==============================================================\n    X = pd.DataFrame(\n        {\n            # [(x_a - x_b)**2 + (y_a - y_b)**2]  => carré de la distance\n            f\"12+{p1}+{p2}\": np.square(mouse_pair[\"A\"][p1] - mouse_pair[\"B\"][p2]).sum(\n                axis=1, skipna=False\n            )\n            for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n            if p1 in avail_A and p2 in avail_B\n        }\n    )\n    X = X.reindex(\n        columns=[\n            f\"12+{p1}+{p2}\"\n            for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n        ],\n        copy=False,\n    )\n\n    to_keep = [\"body_center\", \"nose\", \"tail_tip\", \"head\", \"tail_base\"]\n\n    for mouse_id in [\"A\", \"B\"]:\n        norm_x = pd.concat(\n            [\n                mouse_pair[mouse_id][bp][\"x\"] / arena_size[0]\n                for bp in body_parts_tracked\n                if bp in to_keep\n            ],\n            axis=1,\n        )\n        norm_x.columns = [\n            f\"{bp}_x_norm_{mouse_id}\" for bp in body_parts_tracked if bp in to_keep\n        ]\n\n        norm_y = pd.concat(\n            [\n                mouse_pair[mouse_id][bp][\"y\"] / arena_size[0]\n                for bp in body_parts_tracked\n                if bp in to_keep\n            ],\n            axis=1,\n        )\n        norm_y.columns = [\n            f\"{bp}_y_norm_{mouse_id}\" for bp in body_parts_tracked if bp in to_keep\n        ]\n\n        normed_coords = norm_x.join(norm_y)\n\n        X = X.join(normed_coords)\n\n    # ==============================================================\n    # features n°2: speeds of ears\n    # ==============================================================\n    if (\"A\", \"ear_left\") in mouse_pair.columns and (\n        \"B\",\n        \"ear_left\",\n    ) in mouse_pair.columns:\n        for lag_val in [10, 20, 30]:\n            lag = _scale(lag_val, fps)\n            shA = mouse_pair[\"A\"][\"ear_left\"].shift(lag)\n            shB = mouse_pair[\"B\"][\"ear_left\"].shift(lag)\n            speeds = pd.DataFrame(\n                {\n                    f\"sp_A_{lag_val}\": np.square(mouse_pair[\"A\"][\"ear_left\"] - shA).sum(\n                        axis=1, skipna=False\n                    ),\n                    f\"sp_AB_{lag_val}\": np.square(\n                        mouse_pair[\"A\"][\"ear_left\"] - shB\n                    ).sum(axis=1, skipna=False),\n                    f\"sp_B_{lag_val}\": np.square(mouse_pair[\"B\"][\"ear_left\"] - shB).sum(\n                        axis=1, skipna=False\n                    ),\n                }\n            )\n            X = pd.concat([X, speeds], axis=1)\n\n    # ==============================================================\n    # features n°3:\n    # ==============================================================\n    if \"nose+tail_base\" in X.columns and \"ear_left+ear_right\" in X.columns:\n        X[\"elong\"] = X[\"nose+tail_base\"] / (X[\"ear_left+ear_right\"] + 1e-6)\n\n    # ==============================================================\n    # features n°4:\n    # ==============================================================\n    if all(p in avail_A for p in [\"nose\", \"tail_base\"]) and all(\n        p in avail_B for p in [\"nose\", \"tail_base\"]\n    ):\n        dir_A = mouse_pair[\"A\"][\"nose\"] - mouse_pair[\"A\"][\"tail_base\"]\n        dir_B = mouse_pair[\"B\"][\"nose\"] - mouse_pair[\"B\"][\"tail_base\"]\n        X[\"rel_ori\"] = (dir_A[\"x\"] * dir_B[\"x\"] + dir_A[\"y\"] * dir_B[\"y\"]) / (\n            np.sqrt(dir_A[\"x\"] ** 2 + dir_A[\"y\"] ** 2)\n            * np.sqrt(dir_B[\"x\"] ** 2 + dir_B[\"y\"] ** 2)\n            + 1e-6\n        )\n\n    # ==============================================================\n    # features n°5:\n    # ==============================================================\n    if all(p in avail_A for p in [\"nose\"]) and all(p in avail_B for p in [\"nose\"]):\n        cur = np.square(mouse_pair[\"A\"][\"nose\"] - mouse_pair[\"B\"][\"nose\"]).sum(\n            axis=1, skipna=False\n        )\n        for lag_val in [10, 20, 30]:\n            lag = _scale(lag_val, fps)\n            shA_n = mouse_pair[\"A\"][\"nose\"].shift(lag)\n            shB_n = mouse_pair[\"B\"][\"nose\"].shift(lag)\n            past = np.square(shA_n - shB_n).sum(axis=1, skipna=False)\n            X[f\"appr_{lag_val}\"] = cur - past\n\n    # ==============================================================\n    # features n°6:\n    # ==============================================================\n    if \"body_center\" in avail_A and \"body_center\" in avail_B:\n        cd = np.sqrt(\n            (mouse_pair[\"A\"][\"body_center\"][\"x\"] - mouse_pair[\"B\"][\"body_center\"][\"x\"])\n            ** 2\n            + (\n                mouse_pair[\"A\"][\"body_center\"][\"y\"]\n                - mouse_pair[\"B\"][\"body_center\"][\"y\"]\n            )\n            ** 2\n        )\n        X[\"v_cls\"] = (cd < 5.0).astype(float)\n        X[\"cls\"] = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n        X[\"med\"] = ((cd >= 15.0) & (cd < 30.0)).astype(float)\n        X[\"far\"] = (cd >= 30.0).astype(float)\n\n    # ==============================================================\n    # features n°7\n    # ==============================================================\n    if \"body_center\" in avail_A and \"body_center\" in avail_B:\n        cd_full = np.square(\n            mouse_pair[\"A\"][\"body_center\"] - mouse_pair[\"B\"][\"body_center\"]\n        ).sum(axis=1, skipna=False)\n\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"d_m{w}\"] = cd_full.rolling(ws, **roll).mean()\n            X[f\"d_s{w}\"] = cd_full.rolling(ws, **roll).std()\n            X[f\"d_mn{w}\"] = cd_full.rolling(ws, **roll).min()\n            X[f\"d_mx{w}\"] = cd_full.rolling(ws, **roll).max()\n\n            d_var = cd_full.rolling(ws, **roll).var()\n            X[f\"int{w}\"] = 1 / (1 + d_var)\n\n            Axd = mouse_pair[\"A\"][\"body_center\"][\"x\"].diff()\n            Ayd = mouse_pair[\"A\"][\"body_center\"][\"y\"].diff()\n            Bxd = mouse_pair[\"B\"][\"body_center\"][\"x\"].diff()\n            Byd = mouse_pair[\"B\"][\"body_center\"][\"y\"].diff()\n            coord = Axd * Bxd + Ayd * Byd\n            X[f\"co_m{w}\"] = coord.rolling(ws, **roll).mean()\n            X[f\"co_s{w}\"] = coord.rolling(ws, **roll).std()\n\n    # ==============================================================\n    # features n°8\n    # ==============================================================\n    if \"nose\" in avail_A and \"nose\" in avail_B:\n        nn = np.sqrt(\n            (mouse_pair[\"A\"][\"nose\"][\"x\"] - mouse_pair[\"B\"][\"nose\"][\"x\"]) ** 2\n            + (mouse_pair[\"A\"][\"nose\"][\"y\"] - mouse_pair[\"B\"][\"nose\"][\"y\"]) ** 2\n        )\n        for lag in [10, 20, 40]:\n            l = _scale(lag, fps)\n            X[f\"nn_lg{lag}\"] = nn.shift(l)\n            X[f\"nn_ch{lag}\"] = nn - nn.shift(l)\n            is_cl = (nn < 10.0).astype(float)\n            X[f\"cl_ps{lag}\"] = is_cl.rolling(l, min_periods=1).mean()\n\n    # ==============================================================\n    # features n°10\n    # ==============================================================\n    if \"nose\" in avail_A and \"nose\" in avail_B:\n        cd_full = np.square(mouse_pair[\"A\"][\"nose\"] - mouse_pair[\"B\"][\"nose\"]).sum(\n            axis=1, skipna=False\n        )\n        roll = dict(min_periods=1, center=True)\n        for w in [10, 20, 30]:\n            ws = _scale(lag, fps)\n            X[f\"nn_roll_mean{lag}\"] = cd_full.rolling(ws, **roll).mean()\n            X[f\"nn_roll_median{lag}\"] = cd_full.rolling(ws, **roll).median()\n            X[f\"nn_roll_std{lag}\"] = cd_full.rolling(ws, **roll).std()\n\n    # ==============================================================\n    # features n°11\n    # ==============================================================\n    head_columns_A = [\n        c for c in [\"ear_left\", \"ear_right\", \"nose\", \"neck\"] if c in avail_A\n    ]\n    head_columns_B = [\n        c for c in [\"ear_left\", \"ear_right\", \"nose\", \"neck\"] if c in avail_B\n    ]\n    if len(head_columns_A) > 0 and len(head_columns_B) > 0:\n        mouse_a = mouse_pair[\"A\"][head_columns_A].groupby(level=1, axis=1).mean()\n        mouse_b = mouse_pair[\"B\"][head_columns_B].groupby(level=1, axis=1).mean()\n        global_head_dist = np.sqrt(\n            np.square(mouse_a - mouse_b).sum(axis=1, skipna=False)\n        )\n        roll = dict(min_periods=1, center=True)\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"head_m{w}\"] = global_head_dist.rolling(ws, **roll).mean()\n            X[f\"head_s{w}\"] = global_head_dist.rolling(ws, **roll).std()\n            X[f\"head_mn{w}\"] = global_head_dist.rolling(ws, **roll).min()\n            X[f\"head_mx{w}\"] = global_head_dist.rolling(ws, **roll).max()\n\n    # ==============================================================\n    # features n°12\n    # ==============================================================\n    head_columns = [\n        c for c in [\"ear_left\", \"ear_right\", \"nose\", \"neck\"] if c in avail_A\n    ]\n    body_columns = [\n        c\n        for c in [\"body_center\", \"lateral_right\", \"lateral_left\", \"neck\", \"tail_base\"]\n        if c in avail_B\n    ]\n    if len(head_columns) > 0 and len(body_columns) > 0:\n        mouse_a = mouse_pair[\"A\"][head_columns].groupby(level=1, axis=1).mean()\n        mouse_b = mouse_pair[\"B\"][body_columns].groupby(level=1, axis=1).mean()\n        head_body_dist = np.sqrt(np.square(mouse_a - mouse_b).sum(axis=1, skipna=False))\n        roll = dict(min_periods=1, center=True)\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"body_head_m{w}\"] = head_body_dist.rolling(ws, **roll).mean()\n            X[f\"body_head_s{w}\"] = head_body_dist.rolling(ws, **roll).std()\n            X[f\"body_head_mn{w}\"] = head_body_dist.rolling(ws, **roll).min()\n            X[f\"body_head_mx{w}\"] = head_body_dist.rolling(ws, **roll).max()\n\n    # ==============================================================\n    # features n°13\n    # ==============================================================\n    head_columns = [\n        c for c in [\"ear_left\", \"ear_right\", \"nose\", \"neck\"] if c in avail_A\n    ]\n    tail_columns = [c for c in avail_B if c.startswith(\"tail\")]\n\n    if len(head_columns) > 0 and len(tail_columns) > 0:\n        mouse_a = mouse_pair[\"A\"][head_columns].groupby(level=1, axis=1).mean()\n        mouse_b = mouse_pair[\"B\"][tail_columns].groupby(level=1, axis=1).mean()\n        head_tail_dist = np.sqrt(np.square(mouse_a - mouse_b).sum(axis=1, skipna=False))\n        roll = dict(min_periods=1, center=True)\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"tail_head_m{w}\"] = head_tail_dist.rolling(ws, **roll).mean()\n            X[f\"tail_head_s{w}\"] = head_tail_dist.rolling(ws, **roll).std()\n            X[f\"tail_head_mn{w}\"] = head_tail_dist.rolling(ws, **roll).min()\n            X[f\"tail_head_mx{w}\"] = head_tail_dist.rolling(ws, **roll).max()\n    # ==============================================================\n    # features n°9\n    # ==============================================================\n    if \"body_center\" in avail_A and \"body_center\" in avail_B:\n        Avx = mouse_pair[\"A\"][\"body_center\"][\"x\"].diff()\n        Avy = mouse_pair[\"A\"][\"body_center\"][\"y\"].diff()\n        Bvx = mouse_pair[\"B\"][\"body_center\"][\"x\"].diff()\n        Bvy = mouse_pair[\"B\"][\"body_center\"][\"y\"].diff()\n        val = (Avx * Bvx + Avy * Bvy) / (\n            np.sqrt(Avx**2 + Avy**2) * np.sqrt(Bvx**2 + Bvy**2) + 1e-6\n        )\n\n        for off in [-30, -20, -10, 0, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f\"va_{off}\"] = val.shift(-o)\n\n        w = _scale(30, fps)\n        X[\"int_con\"] = cd_full.rolling(w, min_periods=1, center=True).std() / (\n            cd_full.rolling(w, min_periods=1, center=True).mean() + 1e-6\n        )\n\n        X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n    return X.astype(np.float32, copy=False)","metadata":{"papermill":{"duration":0.051971,"end_time":"2025-10-25T19:26:58.497403","exception":false,"start_time":"2025-10-25T19:26:58.445432","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.425929Z","iopub.execute_input":"2025-12-12T14:43:33.426418Z","iopub.status.idle":"2025-12-12T14:43:33.484264Z","shell.execute_reply.started":"2025-12-12T14:43:33.426378Z","shell.execute_reply":"2025-12-12T14:43:33.483128Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training, validation and submission","metadata":{"papermill":{"duration":0.006521,"end_time":"2025-10-25T19:26:58.511707","exception":false,"start_time":"2025-10-25T19:26:58.505186","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def robustify(submission, dataset, traintest, traintest_directory=None):\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n    old_submission = submission.copy()\n    submission = submission[submission.start_frame < submission.stop_frame]\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped frames with start >= stop\")\n    \n    old_submission = submission.copy()\n    group_list = []\n    for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n        group = group.sort_values('start_frame')\n        mask = np.ones(len(group), dtype=bool)\n        last_stop_frame = 0\n        for i, (_, row) in enumerate(group.iterrows()):\n            if row['start_frame'] < last_stop_frame:\n                mask[i] = False\n            else:\n                last_stop_frame = row['stop_frame']\n        group_list.append(group[mask])\n        \n    submission = pd.concat(group_list)\n    \n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped duplicate frames\")\n        \n    s_list = []\n    for idx, row in dataset.iterrows():\n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'):\n            continue\n        \n        video_id = row['video_id']\n        if (submission.video_id == video_id).any():\n            continue\n        \n        if type(row.behaviors_labeled) != str:\n            continue\n\n        print(f\"Video {video_id} has no predictions.\")\n        \n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n    \n        vid_behaviors = json.loads(row['behaviors_labeled'])\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(',') for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n    \n        start_frame = vid.video_frame.min()\n        stop_frame = vid.video_frame.max() + 1\n    \n        for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n            batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n            for i, (_, action_row) in enumerate(actions.iterrows()):\n                batch_start = start_frame + i * batch_length\n                batch_stop = min(batch_start + batch_length, stop_frame)\n                s_list.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n\n    if len(s_list) > 0:\n        submission = pd.concat([\n            submission,\n            pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n        ])\n        print(\"ERROR: Filled empty videos\")\n\n    submission = submission.reset_index(drop=True)\n    \n    return submission","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.023773,"end_time":"2025-10-25T19:26:58.542199","exception":false,"start_time":"2025-10-25T19:26:58.518426","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.487276Z","iopub.execute_input":"2025-12-12T14:43:33.487610Z","iopub.status.idle":"2025-12-12T14:43:33.510586Z","shell.execute_reply.started":"2025-12-12T14:43:33.487574Z","shell.execute_reply":"2025-12-12T14:43:33.509433Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_multiclass(pred, meta, thresholds):\n    ama = np.argmax(pred.values, axis=1)\n    max_proba = pred.max(axis=1).values\n\n    threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n    action_thresholds = threshold_array[ama]\n\n    ama = np.where(max_proba >= action_thresholds, ama, -1)\n    ama = pd.Series(ama, index=meta.video_frame)\n    \n    changes_mask = (ama != ama.shift(1)).values\n    ama_changes = ama[changes_mask]\n    meta_changes = meta[changes_mask]\n    \n    mask = ama_changes.values >= 0\n    mask[-1] = False\n    \n    submission_part = pd.DataFrame({\n        'video_id': meta_changes['video_id'][mask].values,\n        'agent_id': meta_changes['agent_id'][mask].values,\n        'target_id': meta_changes['target_id'][mask].values,\n        'action': pred.columns[ama_changes[mask].values],\n        'start_frame': ama_changes.index[mask],\n        'stop_frame': ama_changes.index[1:][mask[:-1]]\n    })\n    \n    stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n    stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n    stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n    for i in range(len(submission_part)):\n        video_id = submission_part.video_id.iloc[i]\n        agent_id = submission_part.agent_id.iloc[i]\n        target_id = submission_part.target_id.iloc[i]\n        if stop_video_id[i] != video_id or stop_agent_id[i] != agent_id or stop_target_id[i] != target_id:\n            new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n            submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n\n    return submission_part","metadata":{"papermill":{"duration":0.022133,"end_time":"2025-10-25T19:26:58.571754","exception":false,"start_time":"2025-10-25T19:26:58.549621","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.512082Z","iopub.execute_input":"2025-12-12T14:43:33.512427Z","iopub.status.idle":"2025-12-12T14:43:33.533287Z","shell.execute_reply.started":"2025-12-12T14:43:33.512396Z","shell.execute_reply":"2025-12-12T14:43:33.531892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(\n    body_parts_tracked_str,\n    switch_tr,\n    section,\n    thresholds,\n    drop_body_parts,\n    test,\n    CFG,\n):\n    body_parts_tracked = json.loads(body_parts_tracked_str)\n    if len(body_parts_tracked) > 5:\n        body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n\n    test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n    # print(f\"labs: {test_subset['lab_id'].unique().tolist()}\")\n    \n    generator = generate_mouse_data(\n        CFG=CFG,\n        dataset=test_subset,\n        drop_body_parts=drop_body_parts,\n        traintest=\"test\" if not CFG.debug_predict else \"train\",\n        generate_single=(switch_tr == \"single\"),\n        generate_pair=(switch_tr == \"pair\"),\n    )\n\n    fps_lookup = (\n        test_subset[[\"video_id\", \"frames_per_second\"]]\n        .drop_duplicates(\"video_id\")\n        .set_index(\"video_id\")[\"frames_per_second\"]\n        .to_dict()\n    )\n\n    submission_list = []\n    for switch_te, data_te, meta_te, actions_te in generator:\n        assert switch_te == switch_tr\n        try:\n            fps_i = fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n\n            if switch_te == \"single\":\n                X_te = transform_single(data_te, body_parts_tracked, fps_i, arena_size=(\n                        meta_te[\"arena_width_pixels\"].unique()[0],\n                        meta_te[\"arena_height_pixels\"].unique()[0],\n                    ),)\n                X_te[\"sex\"] = _get_sex(meta_te, column=\"sex\")\n            else:\n                X_te = transform_pair(data_te, body_parts_tracked, fps_i, arena_size=(\n                        meta_te[\"arena_width_pixels\"].unique()[0],\n                        meta_te[\"arena_height_pixels\"].unique()[0],\n                    ),)\n                X_te[\"agent_sex\"] = _get_sex(meta_te, column=\"agent_sex\")\n                X_te[\"target_sex\"] = _get_sex(meta_te, column=\"target_sex\")\n\n            # display(meta_te)\n            X_te[\"lab_id\"] = meta_te[\"lab_id\"].unique()[0]\n            # print(X_te[\"lab_id\"].unique())\n            X_te[\"lab_id\"] = X_te[\"lab_id\"].astype(\"category\")\n            X_te[\"fps\"] = fps_lookup[meta_te[\"video_id\"].unique()[0]]\n            \n            mouse1 = meta_te[\"agent_id\"].unique()[0]\n            mouse2 = meta_te[\"target_id\"].unique()[0]\n            print(\n                switch_tr,\n                f\"{mouse1} -> {mouse2}\",\n                f\"for video n°{meta_te['video_id'].unique()[0]}\" + 100 * \" \",\n            )\n            # print(\"actions:\", actions_te)\n            \n            # print(section, switch_te, meta_te, X_te.shape)\n            pred = pd.DataFrame(index=meta_te.video_frame)\n            for action in actions_te:\n                if isinstance(CFG.model_base_path, dict):    \n                    name = CFG.model_base_path[section]\n                else:\n                    name = CFG.model_base_path\n                model_path = f\"/kaggle/input/mabe2025-models/{name}/{name}/\"\n                \n                if switch_te == \"pair\":\n                    predict_columns = pd.read_pickle(f\"{model_path}/columns_pair_{section}.pkl\")\n                else:\n                    predict_columns = pd.read_pickle(f\"{model_path}/columns_single_{section}.pkl\")\n\n                # print(predict_columns)\n                \n                files = glob.glob(\n                    f\"{model_path}/xgboost/{section}/{action}/*/*_trainer_*.pkl\"\n                )\n                # in case there are multiple files, take the latest one\n                print(action, \"===>\", [file.split(\"/\")[-1] for file in files], 90*\" \", end=\"\\r\")\n                if len(files) == 0:\n                    continue\n\n\n                pred[action] = 0\n                for file in files:\n                    trainer = joblib.load(file)\n                    \n                    test_preds = np.zeros(X_te.shape[0])\n                    for _ind, estimator in enumerate(trainer[\"estimators\"]):\n                        test_preds += estimator.predict_proba(X_te[predict_columns])[:, 1]\n                        \n                    pred[action] += test_preds / len(trainer[\"estimators\"])\n                   \n                pred[action] /= len(files)\n                \n                del trainer\n                gc.collect()\n\n            if pred.shape[1] != 0:\n                submission_part = predict_multiclass(pred, meta_te, thresholds)\n                submission_list.append(submission_part)\n\n        except KeyError:\n            print(\n                f\"KeyError during prediction for action {action}\\n: {KeyError}\"\n            )\n\n    return submission_list\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:47:25.489786Z","iopub.execute_input":"2025-12-12T14:47:25.490132Z","iopub.status.idle":"2025-12-12T14:47:25.507463Z","shell.execute_reply.started":"2025-12-12T14:47:25.490108Z","shell.execute_reply":"2025-12-12T14:47:25.506209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tune_threshold(oof_action, y_action):\n    def objective(trial):\n        threshold = trial.suggest_float(\"threshold\", 0, 1, step=0.01)\n        return f1_score(y_action, (oof_action >= threshold), zero_division=0)\n\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective, n_trials=1000, n_jobs=-1)\n    return study.best_params[\"threshold\"]","metadata":{"papermill":{"duration":0.016204,"end_time":"2025-10-25T19:26:58.594654","exception":false,"start_time":"2025-10-25T19:26:58.57845","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:43:33.534585Z","iopub.execute_input":"2025-12-12T14:43:33.535304Z","iopub.status.idle":"2025-12-12T14:43:33.563022Z","shell.execute_reply.started":"2025-12-12T14:43:33.535266Z","shell.execute_reply":"2025-12-12T14:43:33.561818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef predict_model(\n    CFG, body_parts_tracked_list, drop_body_parts, test, thresholds\n) -> list:\n    submission_list = []\n\n    for section in range(1, len(body_parts_tracked_list)):\n        \n        body_parts_tracked_str = body_parts_tracked_list[section]\n\n        body_parts_tracked = json.loads(body_parts_tracked_str)\n        text = f\"################## SECTION n°{section} ####################\"\n        print(len(text) * \"=\")\n        print(text)\n        print(len(text) * \"=\")\n        print(f\"Processing videos with: {body_parts_tracked}\")\n        print(120 * \"=\")\n\n        # TODO: est-ce vraiment utile ?\n        if len(body_parts_tracked) > 5:\n            body_parts_tracked = [\n                b for b in body_parts_tracked if b not in drop_body_parts\n            ]\n\n\n        # ================================================\n        # single mouse submission\n        # ================================================\n        if f\"{section}\" in thresholds[\"pair\"].keys():\n            temp_submission_list = submit(\n                body_parts_tracked_str,\n                \"pair\",\n                section,\n                thresholds[\"pair\"][f\"{section}\"],\n                drop_body_parts=drop_body_parts,\n                test=test,\n                CFG=CFG,\n                # logger=logger,\n            )\n            submission_list.extend(temp_submission_list)\n\n        # ================================================\n        # single mouse submission\n        # ================================================\n        if f\"{section}\" in thresholds[\"single\"].keys():\n            temp_submission_list = submit(\n                body_parts_tracked_str,\n                \"single\",\n                section,\n                thresholds[\"single\"][f\"{section}\"],\n                drop_body_parts=drop_body_parts,\n                test=test,\n                CFG=CFG,\n                # logger=logger,\n            )\n            submission_list.extend(temp_submission_list)\n\n    return submission_list\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:47:27.988247Z","iopub.execute_input":"2025-12-12T14:47:27.988549Z","iopub.status.idle":"2025-12-12T14:47:27.998155Z","shell.execute_reply.started":"2025-12-12T14:47:27.988529Z","shell.execute_reply":"2025-12-12T14:47:27.996996Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# name = \"ea61607de\"\n# section = \"1\"\n# action = \"approach\"\n# model_path = f\"/kaggle/input/mabe2025-models/{name}/{name}/\"\n# files = glob.glob(f\"{model_path}/xgboost/{section}/{action}/*/*_trainer_*.pkl\", recursive=True)\n# files","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:47:28.212683Z","iopub.execute_input":"2025-12-12T14:47:28.213069Z","iopub.status.idle":"2025-12-12T14:47:28.218397Z","shell.execute_reply.started":"2025-12-12T14:47:28.213020Z","shell.execute_reply":"2025-12-12T14:47:28.217115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if isinstance(CFG.model_base_path, dict):\n    thresholds = {\"single\": {}, \"pair\": {}}\n    for section, name in CFG.model_base_path.items():\n        model_path = f\"/kaggle/input/mabe2025-models/{name}/{name}/\"\n        _thresholds = joblib.load(f\"{model_path}/xgboost/thresholds.pkl\")\n        thresholds[\"single\"][str(section)] = _thresholds[\"single\"].get(str(section), {}).copy()\n        thresholds[\"pair\"][str(section)] = _thresholds[\"pair\"].get(str(section), {}).copy()\n        \nelse:\n    name = CFG.model_base_path\n    model_path = f\"/kaggle/input/mabe2025-models/{name}/{name}/\"\n    thresholds = joblib.load(f\"{CFG.model_base_path}/xgboost/thresholds.pkl\")\n    \n    \nsubmission_list = predict_model(\n    CFG=CFG,\n    body_parts_tracked_list=body_parts_tracked_list,\n    drop_body_parts=drop_body_parts,\n    test=test,\n    thresholds=thresholds,\n)","metadata":{"papermill":{"duration":0.023315,"end_time":"2025-10-25T19:26:58.680526","exception":false,"start_time":"2025-10-25T19:26:58.657211","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:47:28.647819Z","iopub.execute_input":"2025-12-12T14:47:28.648192Z","execution_failed":"2025-12-12T15:12:55.272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.009079,"end_time":"2025-10-25T19:51:34.072884","exception":false,"start_time":"2025-10-25T19:51:34.063805","status":"completed"},"tags":[]}},{"cell_type":"code","source":"ls /kaggle/input/mabe2025-models/ea61607de/ea61607de/","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-12T15:12:55.272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.concat(submission_list)\nsubmission_robust = robustify(submission, test, 'test')\nsubmission_robust.index.name = 'row_id'\nsubmission_robust.to_csv('submission.csv')\nsubmission_robust.head(20)","metadata":{"papermill":{"duration":0.154772,"end_time":"2025-10-25T19:51:34.236441","exception":false,"start_time":"2025-10-25T19:51:34.081669","status":"completed"},"tags":[],"trusted":true,"execution":{"execution_failed":"2025-12-12T15:12:55.272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nsubmission_robust.sort_values([\"video_id\", \"start_frame\"])","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-12T15:12:55.272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_robust.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T14:45:41.664406Z","iopub.status.idle":"2025-12-12T14:45:41.664842Z","shell.execute_reply.started":"2025-12-12T14:45:41.664675Z","shell.execute_reply":"2025-12-12T14:45:41.664697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}