{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13822995,"sourceType":"datasetVersion","datasetId":8802877}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":1494.813501,"end_time":"2025-10-25T19:51:35.870094","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-25T19:26:41.056593","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00962fe9010e42b7a5b123af50d2a026":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b098fa23a3b4031bf88b32b20044978":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"108c3e5db1154f47a5aca68386ed4f24":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"38c36db3be3843b19da108d4cd1f9421":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e9806911d1d489d963f76955d8468c4","IPY_MODEL_e6ad60bc760e4b51885b15335c048a65","IPY_MODEL_6f4723180bf14101b60deeac9fd59e50"],"layout":"IPY_MODEL_00962fe9010e42b7a5b123af50d2a026","tabbable":null,"tooltip":null}},"3e9806911d1d489d963f76955d8468c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_aeb2544f9ad9413b8b71b5f5643b1019","placeholder":"​","style":"IPY_MODEL_ec012e9ac53d477daee0988a35117f0b","tabbable":null,"tooltip":null,"value":"100%"}},"3edd91e1eafb4f6bbf55275ea60b6c97":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f4723180bf14101b60deeac9fd59e50":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e8f95c8096294ce6b6fb2b65c08af536","placeholder":"​","style":"IPY_MODEL_0b098fa23a3b4031bf88b32b20044978","tabbable":null,"tooltip":null,"value":" 863/863 [00:08<00:00, 96.46it/s]"}},"aeb2544f9ad9413b8b71b5f5643b1019":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e6ad60bc760e4b51885b15335c048a65":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_3edd91e1eafb4f6bbf55275ea60b6c97","max":863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_108c3e5db1154f47a5aca68386ed4f24","tabbable":null,"tooltip":null,"value":863}},"e8f95c8096294ce6b6fb2b65c08af536":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec012e9ac53d477daee0988a35117f0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**References**\n- [MABe Nearest Neighbors: The Original ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-nearest-neighbors-the-original)\n- [MABe EDA which makes sense ⭐️⭐️⭐️⭐️⭐️](https://www.kaggle.com/code/ambrosm/mabe-eda-which-makes-sense)\n- [MABe Validated baseline without machine learning](https://www.kaggle.com/code/ambrosm/mabe-validated-baseline-without-machine-learning)\n- [Squeeze GBT](https://www.kaggle.com/code/cody11null/squeeze-gbt)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":0.007766,"end_time":"2025-10-25T19:26:46.411629","exception":false,"start_time":"2025-10-25T19:26:46.403863","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# Imports and configs","metadata":{"papermill":{"duration":0.007171,"end_time":"2025-10-25T19:26:46.426493","exception":false,"start_time":"2025-10-25T19:26:46.419322","status":"completed"},"tags":[]}},{"cell_type":"code","source":"\"\"\"F Beta customized for the data format of the MABe challenge.\"\"\"\n\nimport json\n\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\n\n\nclass HostVisibleError(Exception):\n pass\n\n\ndef single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n label_frames: defaultdict[str, set[int]] = defaultdict(set)\n prediction_frames: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_solution.to_dicts():\n label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n for video in lab_solution['video_id'].unique():\n active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first() # ty: ignore\n active_labels: set[str] = set(json.loads(active_labels))\n predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set)\n\n for row in lab_submission.filter(pl.col('video_id') == video).to_dicts():\n # Since the labels are sparse, we can't evaluate prediction keys not in the active labels.\n if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n continue\n\n new_frames = set(range(row['start_frame'], row['stop_frame']))\n # Ignore truly redundant predictions.\n new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n # A single agent can have multiple targets per frame (ex: evading all other mice) but only one action per target per frame.\n raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n prediction_frames[row['prediction_key']].update(new_frames)\n predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n tps = defaultdict(int)\n fns = defaultdict(int)\n fps = defaultdict(int)\n for key, pred_frames in prediction_frames.items():\n action = key.split('_')[-1]\n matched_label_frames = label_frames[key]\n tps[action] += len(pred_frames.intersection(matched_label_frames))\n fns[action] += len(matched_label_frames.difference(pred_frames))\n fps[action] += len(pred_frames.difference(matched_label_frames))\n\n distinct_actions = set()\n for key, frames in label_frames.items():\n action = key.split('_')[-1]\n distinct_actions.add(action)\n if key not in prediction_frames:\n fns[action] += len(frames)\n\n action_f1s = []\n for action in distinct_actions:\n if tps[action] + fns[action] + fps[action] == 0:\n action_f1s.append(0)\n else:\n action_f1s.append((1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action]))\n return sum(action_f1s) / len(action_f1s)\n\n\ndef mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n \"\"\"\n Doctests:\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 0, 'stop_frame': 10}, # Wrong action\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.500000000000'\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n ... ])\n >>> \"%.12f\" % mouse_fbeta(solution, submission)\n '0.250000000000'\n\n >>> # Overlapping solution events, one prediction matching both.\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 10, 'stop_frame': 20, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 20},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 1.0\n\n >>> solution = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 30, 'stop_frame': 40, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n ... ])\n >>> submission = pd.DataFrame([\n ... {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 40},\n ... ])\n >>> mouse_fbeta(solution, submission)\n 0.6666666666666666\n \"\"\"\n if len(solution) == 0 or len(submission) == 0:\n raise ValueError('Missing solution or submission data')\n\n expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n for col in expected_cols:\n if col not in solution.columns:\n raise ValueError(f'Solution is missing column {col}')\n if col not in submission.columns:\n raise ValueError(f'Submission is missing column {col}')\n\n solution: pl.DataFrame = pl.DataFrame(solution)\n submission: pl.DataFrame = pl.DataFrame(submission)\n assert (solution['start_frame'] <= solution['stop_frame']).all()\n assert (submission['start_frame'] <= submission['stop_frame']).all()\n solution_videos = set(solution['video_id'].unique())\n # Need to align based on video IDs as we can't rely on the row IDs for handling public/private splits.\n submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n solution = solution.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('label_key'),\n )\n submission = submission.with_columns(\n pl.concat_str(\n [\n pl.col('video_id').cast(pl.Utf8),\n pl.col('agent_id').cast(pl.Utf8),\n pl.col('target_id').cast(pl.Utf8),\n pl.col('action'),\n ],\n separator='_',\n ).alias('prediction_key'),\n )\n\n lab_scores = []\n for lab in solution['lab_id'].unique():\n lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n lab_videos = set(lab_solution['video_id'].unique())\n lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n lab_scores.append(single_lab_f1(lab_solution, lab_submission, beta=beta))\n\n return sum(lab_scores) / len(lab_scores)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n \"\"\"\n F1 score for the MABe Challenge\n \"\"\"\n solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n return mouse_fbeta(solution, submission, beta=beta)","metadata":{"_kg_hide-input":true,"papermill":{"duration":2.975797,"end_time":"2025-10-25T19:26:49.408873","exception":false,"start_time":"2025-10-25T19:26:46.433076","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:09:45.331206Z","iopub.execute_input":"2025-12-11T03:09:45.331470Z","iopub.status.idle":"2025-12-11T03:09:46.275381Z","shell.execute_reply.started":"2025-12-11T03:09:45.331449Z","shell.execute_reply":"2025-12-11T03:09:46.274557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.base import clone\nfrom xgboost import XGBClassifier\nfrom tqdm.notebook import tqdm\nfrom transformers import Trainer\nfrom lightgbm import LGBMClassifier\nimport numpy as np\nimport itertools\nimport warnings\nimport optuna\nimport joblib\nimport glob\nimport gc\nimport os\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')","metadata":{"papermill":{"duration":8.578306,"end_time":"2025-10-25T19:26:57.993688","exception":false,"start_time":"2025-10-25T19:26:49.415382","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:09:46.276831Z","iopub.execute_input":"2025-12-11T03:09:46.277247Z","iopub.status.idle":"2025-12-11T03:10:21.235908Z","shell.execute_reply.started":"2025-12-11T03:09:46.277217Z","shell.execute_reply":"2025-12-11T03:10:21.234951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n train_path = \"/kaggle/input/MABe-mouse-behavior-detection/train.csv\"\n test_path = \"/kaggle/input/MABe-mouse-behavior-detection/test.csv\"\n train_annotation_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n train_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n test_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\"\n\n # 修改这里：直接使用完整的模型路径，不需要额外的model_name\n model_path = \"/kaggle/input/kk-lightgbm\"\n model_name = \"lightgbm\" # 设为空字符串\n \n mode = \"submit\"\n \n n_splits = 2\n cv = StratifiedGroupKFold(n_splits)\n\n model = LGBMClassifier(\n random_state=42,\n n_estimators=500,\n learning_rate=0.05,\n max_depth=6,\n num_leaves=31,\n min_child_samples=20,\n subsample=0.8,\n colsample_bytree=0.8,\n reg_alpha=0.1,\n reg_lambda=0.1,\n n_jobs=-1,\n verbosity=-1\n )","metadata":{"papermill":{"duration":0.014723,"end_time":"2025-10-25T19:26:58.043155","exception":false,"start_time":"2025-10-25T19:26:58.028432","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.236800Z","iopub.execute_input":"2025-12-11T03:10:21.237660Z","iopub.status.idle":"2025-12-11T03:10:21.243322Z","shell.execute_reply.started":"2025-12-11T03:10:21.237632Z","shell.execute_reply":"2025-12-11T03:10:21.242436Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{"papermill":{"duration":0.00762,"end_time":"2025-10-25T19:26:58.057724","exception":false,"start_time":"2025-10-25T19:26:58.050104","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv(CFG.train_path)\ntrain['n_mice'] = 4 - train[['mouse1_strain', 'mouse2_strain', 'mouse3_strain', 'mouse4_strain']].isna().sum(axis=1)\ntrain_without_mabe22 = train.query(\"~lab_id.str.startswith('MABe22_')\")\n\ntest = pd.read_csv(CFG.test_path)","metadata":{"papermill":{"duration":0.197857,"end_time":"2025-10-25T19:26:58.262128","exception":false,"start_time":"2025-10-25T19:26:58.064271","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.244198Z","iopub.execute_input":"2025-12-11T03:10:21.244497Z","iopub.status.idle":"2025-12-11T03:10:21.410595Z","shell.execute_reply.started":"2025-12-11T03:10:21.244470Z","shell.execute_reply":"2025-12-11T03:10:21.409674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"body_parts_tracked_list = list(np.unique(train.body_parts_tracked))","metadata":{"papermill":{"duration":0.020077,"end_time":"2025-10-25T19:26:58.288913","exception":false,"start_time":"2025-10-25T19:26:58.268836","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.413574Z","iopub.execute_input":"2025-12-11T03:10:21.413870Z","iopub.status.idle":"2025-12-11T03:10:21.421520Z","shell.execute_reply.started":"2025-12-11T03:10:21.413835Z","shell.execute_reply":"2025-12-11T03:10:21.420464Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Creating solution data","metadata":{"papermill":{"duration":0.008813,"end_time":"2025-10-25T19:26:58.304326","exception":false,"start_time":"2025-10-25T19:26:58.295513","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def create_solution_df(dataset):\n solution = []\n for _, row in tqdm(dataset.iterrows(), total=len(dataset)):\n \n lab_id = row['lab_id']\n if lab_id.startswith('MABe22'): \n continue\n \n video_id = row['video_id']\n path = f\"{CFG.train_annotation_path}/{lab_id}/{video_id}.parquet\"\n try:\n annot = pd.read_parquet(path)\n except FileNotFoundError:\n continue\n \n annot['lab_id'] = lab_id\n annot['video_id'] = video_id\n annot['behaviors_labeled'] = row['behaviors_labeled']\n annot['target_id'] = np.where(annot.target_id != annot.agent_id, annot['target_id'].apply(lambda s: f\"mouse{s}\"), 'self')\n annot['agent_id'] = annot['agent_id'].apply(lambda s: f\"mouse{s}\")\n solution.append(annot)\n \n solution = pd.concat(solution)\n \n return solution\n\nif CFG.mode == \"validate\":\n thresholds = {\n \"single\": {},\n \"pair\": {}\n }\nelse:\n # 加载预训练的阈值文件\n try:\n thresholds = joblib.load(f\"{CFG.model_path}/{CFG.model_name}/thresholds.pkl\")\n print(f\"成功加载阈值文件: {CFG.model_path}/{CFG.model_name}/thresholds.pkl\")\n except Exception as e:\n print(f\"加载阈值文件失败: {e}\")\n # 如果加载失败，使用默认阈值\n thresholds = {\n \"single\": {\"default\": 0.5},\n \"pair\": {\"default\": 0.5}\n }","metadata":{"papermill":{"duration":0.018377,"end_time":"2025-10-25T19:26:58.330166","exception":false,"start_time":"2025-10-25T19:26:58.311789","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.422540Z","iopub.execute_input":"2025-12-11T03:10:21.422857Z","iopub.status.idle":"2025-12-11T03:10:21.452908Z","shell.execute_reply.started":"2025-12-11T03:10:21.422835Z","shell.execute_reply":"2025-12-11T03:10:21.451966Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data generator","metadata":{"papermill":{"duration":0.006479,"end_time":"2025-10-25T19:26:58.343672","exception":false,"start_time":"2025-10-25T19:26:58.337193","status":"completed"},"tags":[]}},{"cell_type":"code","source":"drop_body_parts = [\n 'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n 'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright', \n 'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint'\n]\n\ndef generate_mouse_data(dataset, traintest, traintest_directory=None, generate_single=True, generate_pair=True):\n if traintest_directory is None:\n traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n \n for _, row in dataset.iterrows():\n lab_id = row.lab_id\n if lab_id.startswith('MABe22') or type(row.behaviors_labeled) != str: \n continue\n \n video_id = row.video_id\n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n vid = pd.read_parquet(path)\n if len(np.unique(vid.bodypart)) > 5:\n vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n \n del vid\n gc.collect()\n \n pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n pvid /= row.pix_per_cm_approx\n\n vid_behaviors = json.loads(row.behaviors_labeled)\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n if traintest == 'train':\n try:\n annot = pd.read_parquet(path.replace('train_tracking', 'train_annotation'))\n except FileNotFoundError:\n continue\n\n if generate_single:\n vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n try:\n mouse_id = int(mouse_id_str[-1])\n vid_agent_actions = np.unique(vid_behaviors_subset.query(\"agent == @mouse_id_str\").action)\n single_mouse = pvid.loc[:, mouse_id]\n assert len(single_mouse) == len(pvid)\n single_mouse_meta = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': mouse_id_str,\n 'target_id': 'self',\n 'video_frame': single_mouse.index\n })\n if traintest == 'train':\n single_mouse_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=single_mouse.index)\n annot_subset = annot.query(\"(agent_id == @mouse_id) & (target_id == @mouse_id)\")\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n single_mouse_label.loc[annot_row['start_frame']:annot_row['stop_frame'], annot_row.action] = 1.0\n yield 'single', single_mouse, single_mouse_meta, single_mouse_label\n else:\n yield 'single', single_mouse, single_mouse_meta, vid_agent_actions\n except KeyError:\n pass\n\n if generate_pair:\n vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n if len(vid_behaviors_subset) > 0:\n for agent, target in itertools.permutations(np.unique(pvid.columns.get_level_values('mouse_id')), 2): # int8\n agent_str = f\"mouse{agent}\"\n target_str = f\"mouse{target}\"\n vid_agent_actions = np.unique(vid_behaviors_subset.query(\"(agent == @agent_str) & (target == @target_str)\").action)\n mouse_pair = pd.concat([pvid[agent], pvid[target]], axis=1, keys=['A', 'B'])\n assert len(mouse_pair) == len(pvid)\n mouse_pair_meta = pd.DataFrame({\n 'video_id': video_id,\n 'agent_id': agent_str,\n 'target_id': target_str,\n 'video_frame': mouse_pair.index\n })\n if traintest == 'train':\n mouse_pair_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=mouse_pair.index)\n annot_subset = annot.query(\"(agent_id == @agent) & (target_id == @target)\")\n for i in range(len(annot_subset)):\n annot_row = annot_subset.iloc[i]\n mouse_pair_label.loc[annot_row['start_frame']:annot_row['stop_frame'], annot_row.action] = 1.0\n yield 'pair', mouse_pair, mouse_pair_meta, mouse_pair_label\n else:\n yield 'pair', mouse_pair, mouse_pair_meta, vid_agent_actions","metadata":{"papermill":{"duration":0.028134,"end_time":"2025-10-25T19:26:58.378526","exception":false,"start_time":"2025-10-25T19:26:58.350392","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.454083Z","iopub.execute_input":"2025-12-11T03:10:21.454401Z","iopub.status.idle":"2025-12-11T03:10:21.469787Z","shell.execute_reply.started":"2025-12-11T03:10:21.454372Z","shell.execute_reply":"2025-12-11T03:10:21.468962Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transforming coordinates","metadata":{"papermill":{"duration":0.006144,"end_time":"2025-10-25T19:26:58.391332","exception":false,"start_time":"2025-10-25T19:26:58.385188","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def safe_rolling(series, window, func, min_periods=None):\n if min_periods is None:\n min_periods = max(1, window // 4)\n return series.rolling(window, min_periods=min_periods, center=True).apply(func, raw=True)\n\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n if n_frames_at_30fps == 0:\n return 0\n s = 1 if n_frames_at_30fps > 0 else -1\n mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n return s * mag\n\ndef _fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n if 'frames_per_second' in meta_df.columns and pd.notnull(meta_df['frames_per_second']).any():\n return float(meta_df['frames_per_second'].iloc[0])\n vid = meta_df['video_id'].iloc[0]\n return float(fallback_lookup.get(vid, default_fps))\n\ndef add_curvature_features(X, center_x, center_y, fps):\n vel_x = center_x.diff()\n vel_y = center_y.diff()\n acc_x = vel_x.diff()\n acc_y = vel_y.diff()\n\n cross_prod = vel_x * acc_y - vel_y * acc_x\n vel_mag = np.sqrt(vel_x**2 + vel_y**2)\n curvature = np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n\n for w in [25, 50, 75]:\n ws = _scale(w, fps)\n X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 5)).mean()\n\n angle = np.arctan2(vel_y, vel_x)\n angle_change = np.abs(angle.diff())\n w = 30\n ws = _scale(w, fps)\n X[f'turn_rate_{w}'] = angle_change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n\n return X\n\ndef add_multiscale_features(X, center_x, center_y, fps):\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n\n scales = [20, 40, 60, 80]\n for scale in scales:\n ws = _scale(scale, fps)\n if len(speed) >= ws:\n X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n\n if len(scales) >= 2 and f'sp_m{scales[0]}' in X.columns and f'sp_m{scales[-1]}' in X.columns:\n X['sp_ratio'] = X[f'sp_m{scales[0]}'] / (X[f'sp_m{scales[-1]}'] + 1e-6)\n\n return X\n\ndef add_state_features(X, center_x, center_y, fps):\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n w_ma = _scale(15, fps)\n speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n\n try:\n bins = [-np.inf, 0.5 * fps, 2.0 * fps, 5.0 * fps, np.inf]\n speed_states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n\n for window in [20, 40, 60, 80]:\n ws = _scale(window, fps)\n if len(speed_states) >= ws:\n for state in [0, 1, 2, 3]:\n X[f's{state}_{window}'] = (\n (speed_states == state).astype(float)\n .rolling(ws, min_periods=max(1, ws // 5)).mean()\n )\n state_changes = (speed_states != speed_states.shift(1)).astype(float)\n X[f'trans_{window}'] = state_changes.rolling(ws, min_periods=max(1, ws // 5)).sum()\n except Exception:\n pass\n\n return X\n\ndef add_longrange_features(X, center_x, center_y, fps):\n for window in [30, 60, 120]:\n ws = _scale(window, fps)\n if len(center_x) >= ws:\n X[f'x_ml{window}'] = center_x.rolling(ws, min_periods=max(5, ws // 6)).mean()\n X[f'y_ml{window}'] = center_y.rolling(ws, min_periods=max(5, ws // 6)).mean()\n\n for span in [30, 60, 120]:\n s = _scale(span, fps)\n X[f'x_e{span}'] = center_x.ewm(span=s, min_periods=1).mean()\n X[f'y_e{span}'] = center_y.ewm(span=s, min_periods=1).mean()\n\n speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps) # cm/s\n for window in [30, 60, 120]:\n ws = _scale(window, fps)\n if len(speed) >= ws:\n X[f'sp_pct{window}'] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n\n return X\n\ndef add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n if 'body_center' not in avail_A or 'body_center' not in avail_B:\n return X\n\n rel_x = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n rel_y = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n A_vx = mouse_pair['A']['body_center']['x'].diff()\n A_vy = mouse_pair['A']['body_center']['y'].diff()\n B_vx = mouse_pair['B']['body_center']['x'].diff()\n B_vy = mouse_pair['B']['body_center']['y'].diff()\n\n A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n for window in [30, 60]:\n ws = _scale(window, fps)\n X[f'A_ld{window}'] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n X[f'B_ld{window}'] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n approach = -rel_dist.diff()\n chase = approach * B_lead\n w = 30\n ws = _scale(w, fps)\n X[f'chase_{w}'] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n for window in [60, 120]:\n ws = _scale(window, fps)\n A_sp = np.sqrt(A_vx**2 + A_vy**2)\n B_sp = np.sqrt(B_vx**2 + B_vy**2)\n X[f'sp_cor{window}'] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n\n return X","metadata":{"papermill":{"duration":0.038766,"end_time":"2025-10-25T19:26:58.43742","exception":false,"start_time":"2025-10-25T19:26:58.398654","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.470963Z","iopub.execute_input":"2025-12-11T03:10:21.471210Z","iopub.status.idle":"2025-12-11T03:10:21.496994Z","shell.execute_reply.started":"2025-12-11T03:10:21.471191Z","shell.execute_reply":"2025-12-11T03:10:21.496068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transform_single(single_mouse, body_parts_tracked, fps):\n available_body_parts = single_mouse.columns.get_level_values(0)\n\n X = pd.DataFrame({\n f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n if p1 in available_body_parts and p2 in available_body_parts\n })\n X = X.reindex(columns=[f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(body_parts_tracked, 2)], copy=False)\n\n if all(p in single_mouse.columns for p in ['ear_left', 'ear_right', 'tail_base']):\n lag = _scale(10, fps)\n shifted = single_mouse[['ear_left', 'ear_right', 'tail_base']].shift(lag)\n speeds = pd.DataFrame({\n 'sp_lf': np.square(single_mouse['ear_left'] - shifted['ear_left']).sum(axis=1, skipna=False),\n 'sp_rt': np.square(single_mouse['ear_right'] - shifted['ear_right']).sum(axis=1, skipna=False),\n 'sp_lf2': np.square(single_mouse['ear_left'] - shifted['tail_base']).sum(axis=1, skipna=False),\n 'sp_rt2': np.square(single_mouse['ear_right'] - shifted['tail_base']).sum(axis=1, skipna=False),\n })\n X = pd.concat([X, speeds], axis=1)\n\n if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n if all(p in available_body_parts for p in ['nose', 'body_center', 'tail_base']):\n v1 = single_mouse['nose'] - single_mouse['body_center']\n v2 = single_mouse['tail_base'] - single_mouse['body_center']\n X['body_ang'] = (v1['x'] * v2['x'] + v1['y'] * v2['y']) / (\n np.sqrt(v1['x']**2 + v1['y']**2) * np.sqrt(v2['x']**2 + v2['y']**2) + 1e-6)\n\n if 'body_center' in available_body_parts:\n cx = single_mouse['body_center']['x']\n cy = single_mouse['body_center']['y']\n\n for w in [5, 15, 30, 60]:\n ws = _scale(w, fps)\n roll = dict(min_periods=1, center=True)\n X[f'cx_m{w}'] = cx.rolling(ws, **roll).mean()\n X[f'cy_m{w}'] = cy.rolling(ws, **roll).mean()\n X[f'cx_s{w}'] = cx.rolling(ws, **roll).std()\n X[f'cy_s{w}'] = cy.rolling(ws, **roll).std()\n X[f'x_rng{w}'] = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n X[f'y_rng{w}'] = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n X[f'disp{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).sum()**2 +\n cy.diff().rolling(ws, min_periods=1).sum()**2)\n X[f'act{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).var() +\n cy.diff().rolling(ws, min_periods=1).var())\n\n X = add_curvature_features(X, cx, cy, fps)\n X = add_multiscale_features(X, cx, cy, fps)\n X = add_state_features(X, cx, cy, fps)\n X = add_longrange_features(X, cx, cy, fps)\n\n if all(p in available_body_parts for p in ['nose', 'tail_base']):\n nt_dist = np.sqrt((single_mouse['nose']['x'] - single_mouse['tail_base']['x'])**2 +\n (single_mouse['nose']['y'] - single_mouse['tail_base']['y'])**2)\n for lag in [10, 20, 40]:\n l = _scale(lag, fps)\n X[f'nt_lg{lag}'] = nt_dist.shift(l)\n X[f'nt_df{lag}'] = nt_dist - nt_dist.shift(l)\n\n if all(p in available_body_parts for p in ['ear_left', 'ear_right']):\n ear_d = np.sqrt((single_mouse['ear_left']['x'] - single_mouse['ear_right']['x'])**2 +\n (single_mouse['ear_left']['y'] - single_mouse['ear_right']['y'])**2)\n for off in [-30, -20, -10, 10, 20, 30]:\n o = _scale_signed(off, fps)\n X[f'ear_o{off}'] = ear_d.shift(-o)\n w = _scale(30, fps)\n X['ear_con'] = ear_d.rolling(w, min_periods=1, center=True).std() / \\\n (ear_d.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n return X.astype(np.float32, copy=False)\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n avail_A = mouse_pair['A'].columns.get_level_values(0)\n avail_B = mouse_pair['B'].columns.get_level_values(0)\n\n X = pd.DataFrame({\n f\"12+{p1}+{p2}\": np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n if p1 in avail_A and p2 in avail_B\n })\n X = X.reindex(columns=[f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(body_parts_tracked, repeat=2)], copy=False)\n\n if ('A', 'ear_left') in mouse_pair.columns and ('B', 'ear_left') in mouse_pair.columns:\n lag = _scale(10, fps)\n shA = mouse_pair['A']['ear_left'].shift(lag)\n shB = mouse_pair['B']['ear_left'].shift(lag)\n speeds = pd.DataFrame({\n 'sp_A': np.square(mouse_pair['A']['ear_left'] - shA).sum(axis=1, skipna=False),\n 'sp_AB': np.square(mouse_pair['A']['ear_left'] - shB).sum(axis=1, skipna=False),\n 'sp_B': np.square(mouse_pair['B']['ear_left'] - shB).sum(axis=1, skipna=False),\n })\n X = pd.concat([X, speeds], axis=1)\n\n if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n dir_A = mouse_pair['A']['nose'] - mouse_pair['A']['tail_base']\n dir_B = mouse_pair['B']['nose'] - mouse_pair['B']['tail_base']\n X['rel_ori'] = (dir_A['x'] * dir_B['x'] + dir_A['y'] * dir_B['y']) / (\n np.sqrt(dir_A['x']**2 + dir_A['y']**2) * np.sqrt(dir_B['x']**2 + dir_B['y']**2) + 1e-6)\n\n if all(p in avail_A for p in ['nose']) and all(p in avail_B for p in ['nose']):\n cur = np.square(mouse_pair['A']['nose'] - mouse_pair['B']['nose']).sum(axis=1, skipna=False)\n lag = _scale(10, fps)\n shA_n = mouse_pair['A']['nose'].shift(lag)\n shB_n = mouse_pair['B']['nose'].shift(lag)\n past = np.square(shA_n - shB_n).sum(axis=1, skipna=False)\n X['appr'] = cur - past\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n cd = np.sqrt((mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x'])**2 +\n (mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y'])**2)\n X['v_cls'] = (cd < 5.0).astype(float)\n X['cls'] = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n X['med'] = ((cd >= 15.0) & (cd < 30.0)).astype(float)\n X['far'] = (cd >= 30.0).astype(float)\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n cd_full = np.square(mouse_pair['A']['body_center'] - mouse_pair['B']['body_center']).sum(axis=1, skipna=False)\n\n for w in [5, 15, 30, 60]:\n ws = _scale(w, fps)\n roll = dict(min_periods=1, center=True)\n X[f'd_m{w}'] = cd_full.rolling(ws, **roll).mean()\n X[f'd_s{w}'] = cd_full.rolling(ws, **roll).std()\n X[f'd_mn{w}'] = cd_full.rolling(ws, **roll).min()\n X[f'd_mx{w}'] = cd_full.rolling(ws, **roll).max()\n\n d_var = cd_full.rolling(ws, **roll).var()\n X[f'int{w}'] = 1 / (1 + d_var)\n\n Axd = mouse_pair['A']['body_center']['x'].diff()\n Ayd = mouse_pair['A']['body_center']['y'].diff()\n Bxd = mouse_pair['B']['body_center']['x'].diff()\n Byd = mouse_pair['B']['body_center']['y'].diff()\n coord = Axd * Bxd + Ayd * Byd\n X[f'co_m{w}'] = coord.rolling(ws, **roll).mean()\n X[f'co_s{w}'] = coord.rolling(ws, **roll).std()\n\n if 'nose' in avail_A and 'nose' in avail_B:\n nn = np.sqrt((mouse_pair['A']['nose']['x'] - mouse_pair['B']['nose']['x'])**2 +\n (mouse_pair['A']['nose']['y'] - mouse_pair['B']['nose']['y'])**2)\n for lag in [10, 20, 40]:\n l = _scale(lag, fps)\n X[f'nn_lg{lag}'] = nn.shift(l)\n X[f'nn_ch{lag}'] = nn - nn.shift(l)\n is_cl = (nn < 10.0).astype(float)\n X[f'cl_ps{lag}'] = is_cl.rolling(l, min_periods=1).mean()\n\n if 'body_center' in avail_A and 'body_center' in avail_B:\n Avx = mouse_pair['A']['body_center']['x'].diff()\n Avy = mouse_pair['A']['body_center']['y'].diff()\n Bvx = mouse_pair['B']['body_center']['x'].diff()\n Bvy = mouse_pair['B']['body_center']['y'].diff()\n val = (Avx * Bvx + Avy * Bvy) / (np.sqrt(Avx**2 + Avy**2) * np.sqrt(Bvx**2 + Bvy**2) + 1e-6)\n\n for off in [-30, -20, -10, 0, 10, 20, 30]:\n o = _scale_signed(off, fps)\n X[f'va_{off}'] = val.shift(-o)\n\n w = _scale(30, fps)\n X['int_con'] = cd_full.rolling(w, min_periods=1, center=True).std() / \\\n (cd_full.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n return X.astype(np.float32, copy=False)","metadata":{"papermill":{"duration":0.051971,"end_time":"2025-10-25T19:26:58.497403","exception":false,"start_time":"2025-10-25T19:26:58.445432","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.498674Z","iopub.execute_input":"2025-12-11T03:10:21.499190Z","iopub.status.idle":"2025-12-11T03:10:21.532962Z","shell.execute_reply.started":"2025-12-11T03:10:21.499167Z","shell.execute_reply":"2025-12-11T03:10:21.532292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training, validation and submission","metadata":{"papermill":{"duration":0.006521,"end_time":"2025-10-25T19:26:58.511707","exception":false,"start_time":"2025-10-25T19:26:58.505186","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\n\ndef robustify(submission, dataset, traintest, traintest_directory=None):\n if traintest_directory is None:\n traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n \n old_submission = submission.copy()\n \n # --------------------------\n # 核心修复：将 start_frame/stop_frame 转为整数\n # --------------------------\n # 处理字符串转整数（兼容 \"123\" 这类字符串，无效值转为 NaN 后填充为 0）\n submission['start_frame'] = pd.to_numeric(submission['start_frame'], errors='coerce').fillna(0).astype(int)\n submission['stop_frame'] = pd.to_numeric(submission['stop_frame'], errors='coerce').fillna(0).astype(int)\n \n # 过滤 start >= stop 的无效行（现在是整数比较，无类型错误）\n submission = submission[submission.start_frame < submission.stop_frame]\n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped frames with start >= stop\")\n \n old_submission = submission.copy()\n group_list = []\n # 按 video_id + agent_id + target_id 分组，避免跨主体的帧冲突\n for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n group = group.sort_values('start_frame') # 按开始帧排序\n mask = np.ones(len(group), dtype=bool)\n last_stop_frame = 0\n for i, (_, row) in enumerate(group.iterrows()):\n # 现在 start_frame 是整数，可正常比较\n if row['start_frame'] < last_stop_frame:\n mask[i] = False # 过滤重叠的动作（当前开始帧 < 上一个结束帧）\n else:\n last_stop_frame = row['stop_frame'] # 更新上一个动作的结束帧\n group_list.append(group[mask])\n \n submission = pd.concat(group_list)\n \n if len(submission) != len(old_submission):\n print(\"ERROR: Dropped duplicate frames\")\n \n s_list = []\n # 处理无预测结果的视频，填充默认动作帧\n for idx, row in dataset.iterrows():\n lab_id = row['lab_id']\n if lab_id.startswith('MABe22'):\n continue\n \n video_id = row['video_id']\n if (submission.video_id == video_id).any():\n continue # 已有预测结果，跳过\n \n if type(row.behaviors_labeled) != str:\n continue\n\n print(f\"Video {video_id} has no predictions.\")\n \n path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n vid = pd.read_parquet(path)\n \n vid_behaviors = json.loads(row['behaviors_labeled'])\n vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n vid_behaviors = [b.split(',') for b in vid_behaviors]\n vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n \n start_frame = vid.video_frame.min()\n stop_frame = vid.video_frame.max() + 1\n \n # 按 agent + target 分组，均匀分配动作帧\n for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n for i, (_, action_row) in enumerate(actions.iterrows()):\n batch_start = start_frame + i * batch_length\n batch_stop = min(batch_start + batch_length, stop_frame)\n s_list.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n\n if len(s_list) > 0:\n # 新增的填充行，start_frame/stop_frame 本身是整数，无需转换\n submission = pd.concat([\n submission,\n pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n ])\n print(\"ERROR: Filled empty videos\")\n\n submission = submission.reset_index(drop=True)\n \n return submission","metadata":{"_kg_hide-input":false,"papermill":{"duration":0.023773,"end_time":"2025-10-25T19:26:58.542199","exception":false,"start_time":"2025-10-25T19:26:58.518426","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.534316Z","iopub.execute_input":"2025-12-11T03:10:21.534657Z","iopub.status.idle":"2025-12-11T03:10:21.551356Z","shell.execute_reply.started":"2025-12-11T03:10:21.534636Z","shell.execute_reply":"2025-12-11T03:10:21.550557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_multiclass(pred, meta, thresholds):\n ama = np.argmax(pred.values, axis=1)\n max_proba = pred.max(axis=1).values\n\n threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n action_thresholds = threshold_array[ama]\n\n ama = np.where(max_proba >= action_thresholds, ama, -1)\n ama = pd.Series(ama, index=meta.video_frame)\n \n changes_mask = (ama != ama.shift(1)).values\n ama_changes = ama[changes_mask]\n meta_changes = meta[changes_mask]\n \n mask = ama_changes.values >= 0\n mask[-1] = False\n \n submission_part = pd.DataFrame({\n 'video_id': meta_changes['video_id'][mask].values,\n 'agent_id': meta_changes['agent_id'][mask].values,\n 'target_id': meta_changes['target_id'][mask].values,\n 'action': pred.columns[ama_changes[mask].values],\n 'start_frame': ama_changes.index[mask],\n 'stop_frame': ama_changes.index[1:][mask[:-1]]\n })\n \n stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n for i in range(len(submission_part)):\n video_id = submission_part.video_id.iloc[i]\n agent_id = submission_part.agent_id.iloc[i]\n target_id = submission_part.target_id.iloc[i]\n if stop_video_id[i] != video_id or stop_agent_id[i] != agent_id or stop_target_id[i] != target_id:\n new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n\n return submission_part","metadata":{"papermill":{"duration":0.022133,"end_time":"2025-10-25T19:26:58.571754","exception":false,"start_time":"2025-10-25T19:26:58.549621","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.552241Z","iopub.execute_input":"2025-12-11T03:10:21.552564Z","iopub.status.idle":"2025-12-11T03:10:21.564901Z","shell.execute_reply.started":"2025-12-11T03:10:21.552541Z","shell.execute_reply":"2025-12-11T03:10:21.564177Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tune_threshold(oof_action, y_action):\n def objective(trial):\n threshold = trial.suggest_float(\"threshold\", 0, 1, step=0.01)\n return f1_score(y_action, (oof_action >= threshold), zero_division=0)\n\n study = optuna.create_study(direction=\"maximize\")\n study.optimize(objective, n_trials=1000, n_jobs=-1)\n return study.best_params[\"threshold\"]","metadata":{"papermill":{"duration":0.016204,"end_time":"2025-10-25T19:26:58.594654","exception":false,"start_time":"2025-10-25T19:26:58.57845","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.565837Z","iopub.execute_input":"2025-12-11T03:10:21.566145Z","iopub.status.idle":"2025-12-11T03:10:21.579908Z","shell.execute_reply.started":"2025-12-11T03:10:21.566119Z","shell.execute_reply":"2025-12-11T03:10:21.579187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cross_validate_classifier_lgb(X, label, meta, body_parts_tracked_str, section):\n \"\"\"\n 简化版LightGBM交叉验证，去掉复杂的超参数优化\n \"\"\"\n os.makedirs(f\"{CFG.model_name}/{section}\", exist_ok=True)\n \n oof = pd.DataFrame(index=meta.video_frame)\n f1_list = []\n submission_list = []\n thresholds = {}\n\n for action in label.columns:\n action_mask = ~label[action].isna().values\n y_action = label[action][action_mask].values.astype(int)\n X_action = X[action_mask]\n groups_action = meta.video_id[action_mask]\n\n if len(np.unique(groups_action)) < 2:\n continue\n\n if not (y_action == 0).all():\n try:\n # 简化的LightGBM参数\n model_params = {\n 'n_estimators': 500, # 减少树的数量以加快训练\n 'learning_rate': 0.05,\n 'max_depth': 6,\n 'num_leaves': 31,\n 'min_child_samples': 20,\n 'subsample': 0.8,\n 'colsample_bytree': 0.8,\n 'reg_alpha': 0.1,\n 'reg_lambda': 0.1,\n 'n_jobs': -1,\n 'random_state': 42,\n 'verbosity': -1\n }\n \n # 改为2折交叉验证\n cv = GroupKFold(n_splits=min(2, len(np.unique(groups_action))))\n oof_action = np.zeros(len(y_action))\n models = []\n \n for train_idx, valid_idx in cv.split(X_action, y_action, groups_action):\n model = LGBMClassifier(**model_params)\n model.fit(\n X_action.iloc[train_idx], y_action[train_idx]\n )\n oof_action[valid_idx] = model.predict_proba(X_action.iloc[valid_idx])[:, 1]\n models.append(model)\n \n # 使用简单的阈值优化\n threshold = tune_threshold(oof_action, y_action)\n thresholds[action] = threshold\n \n f1 = f1_score(y_action, (oof_action >= threshold), zero_division=0)\n f1_list.append((body_parts_tracked_str, action, f1))\n print(f\"\\tF1: {f1:.4f} (阈值: {threshold:.3f}) - 动作: {action}\")\n \n # 保存模型\n model_dir = f\"{CFG.model_name}/{section}/{action}\"\n os.makedirs(model_dir, exist_ok=True)\n joblib.dump(models, f\"{model_dir}/lgb_trainer.pkl\")\n\n except Exception as e:\n print(f\"\\t训练失败 {action}: {e}\")\n oof_action = np.zeros(len(y_action))\n else:\n oof_action = np.zeros(len(y_action))\n thresholds[action] = 0.5\n\n oof_column = np.zeros(len(label))\n oof_column[action_mask] = oof_action\n oof[action] = oof_column\n\n del oof_action, action_mask, X_action, y_action, groups_action\n gc.collect()\n\n # 使用原来的predict_multiclass而不是improved版本\n submission_part = predict_multiclass(oof, meta, thresholds)\n submission_list.append(submission_part)\n\n return submission_list, f1_list, thresholds","metadata":{"papermill":{"duration":0.021154,"end_time":"2025-10-25T19:26:58.624019","exception":false,"start_time":"2025-10-25T19:26:58.602865","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.580850Z","iopub.execute_input":"2025-12-11T03:10:21.581626Z","iopub.status.idle":"2025-12-11T03:10:21.602333Z","shell.execute_reply.started":"2025-12-11T03:10:21.581603Z","shell.execute_reply":"2025-12-11T03:10:21.601514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport hashlib\nimport numpy as np\nimport pandas as pd\nimport json\nimport gc\nfrom concurrent.futures import ProcessPoolExecutor, as_completed\nimport joblib\n\ndef submit_improved(body_parts_tracked_str, switch_tr, section, thresholds, max_workers=2):\n \"\"\"\n 修复的submit函数，解决缓存和并行处理问题\n \"\"\"\n body_parts_tracked = json.loads(body_parts_tracked_str)\n if len(body_parts_tracked) > 5:\n body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n\n test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n \n print(f\"\\t测试子集大小: {len(test_subset)}\")\n \n if len(test_subset) == 0:\n print(f\"\\t没有找到匹配的测试数据\")\n return []\n \n # 预先创建FPS查找表\n fps_lookup = (\n test_subset[['video_id', 'frames_per_second']]\n .drop_duplicates('video_id')\n .set_index('video_id')['frames_per_second']\n .to_dict()\n )\n \n # 预先加载所有相关模型到内存\n models_cache = {}\n all_actions = set()\n \n for _, row in test_subset.iterrows():\n if type(row.behaviors_labeled) == str:\n try:\n behaviors = json.loads(row.behaviors_labeled)\n for behavior in behaviors:\n if isinstance(behavior, str) and ',' in behavior:\n parts = behavior.split(',')\n if len(parts) >= 3:\n action = parts[2]\n else:\n action = behavior\n else:\n action = str(behavior)\n all_actions.add(action)\n except Exception as e:\n print(f\"\\t解析行为标签失败: {e}\")\n continue\n \n print(f\"\\t预加载 {len(all_actions)} 个动作的模型: {list(all_actions)}\")\n \n loaded_actions = 0\n for action in all_actions:\n try:\n # 修复路径：直接使用CFG.model_path，不拼接model_name\n model_path_pattern = f\"{CFG.model_path}/{section}/{action}/lgb_trainer.pkl\"\n model_files = glob.glob(model_path_pattern)\n \n if not model_files:\n # 如果找不到，尝试其他可能的路径格式\n model_path_pattern2 = f\"/kaggle/input/kk-lightgbm/lightgbm/{section}/{action}/lgb_trainer.pkl\"\n model_files = glob.glob(model_path_pattern2)\n \n if not model_files:\n # 再尝试一种路径格式\n model_path_pattern3 = f\"/kaggle/input/kk-lightgbm/{section}/{action}/lgb_trainer.pkl\"\n model_files = glob.glob(model_path_pattern3)\n \n if model_files:\n models = joblib.load(model_files[0])\n models_cache[action] = models\n loaded_actions += 1\n print(f\"\\t✓ 加载模型: {action} (路径: {model_files[0]})\")\n else:\n print(f\"\\t✗ 未找到模型: {action}，尝试的路径: {model_path_pattern}\")\n except Exception as e:\n print(f\"\\t✗ 加载模型 {action} 失败: {e}\")\n \n print(f\"\\t成功加载 {loaded_actions}/{len(all_actions)} 个动作的模型\")\n \n if loaded_actions == 0:\n print(f\"\\t警告: 没有加载任何模型，跳过该部分\")\n return []\n \n def process_single_video(row):\n \"\"\"处理单个视频的预测任务\"\"\"\n try:\n lab_id = row['lab_id']\n video_id = row['video_id']\n \n print(f\"\\t处理视频: {video_id}\")\n \n # 生成该视频的数据 - 使用正确的参数\n video_data = pd.DataFrame([row])\n generator = generate_mouse_data(\n video_data,\n 'test',\n traintest_directory=CFG.test_tracking_path,\n generate_single=(switch_tr == 'single'),\n generate_pair=(switch_tr == 'pair')\n )\n \n video_submissions = []\n processed_count = 0\n \n for switch_te, data_te, meta_te, actions_te in generator:\n try:\n processed_count += 1\n print(f\"\\t 处理 {switch_te} 数据, 动作: {list(actions_te)}\")\n \n # 确保meta_te有正确的列\n if 'video_frame' not in meta_te.columns:\n print(f\"\\t 错误: meta_te 缺少 video_frame 列\")\n continue\n \n # 直接计算特征，不使用缓存（避免缓存问题）\n fps_i = _fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n \n if switch_te == 'single':\n X_te = transform_single(data_te, body_parts_tracked, fps_i).astype(np.float32)\n else:\n X_te = transform_pair(data_te, body_parts_tracked, fps_i).astype(np.float32)\n \n # 清理数据\n del data_te\n gc.collect()\n \n # 检查特征维度\n if X_te.shape[0] == 0:\n print(f\"\\t 警告: 特征矩阵为空\")\n continue\n \n # 预测所有相关动作\n pred = pd.DataFrame(index=meta_te.video_frame)\n predicted_actions = 0\n \n for action in actions_te:\n if action in models_cache:\n try:\n models = models_cache[action]\n # 多折预测取平均概率\n probas = []\n for model in models:\n prob = model.predict_proba(X_te)[:, 1]\n probas.append(prob)\n \n if probas:\n mean_prob = np.mean(probas, axis=0)\n pred[action] = mean_prob\n predicted_actions += 1\n \n # 调试信息\n prob_mean = np.mean(mean_prob)\n prob_max = np.max(mean_prob)\n threshold = thresholds.get(action, 0.27)\n print(f\"\\t {action}: 平均概率={prob_mean:.4f}, 最大概率={prob_max:.4f}, 阈值={threshold}\")\n except Exception as e:\n print(f\"\\t 预测动作 {action} 失败: {e}\")\n continue\n \n print(f\"\\t 成功预测 {predicted_actions}/{len(actions_te)} 个动作\")\n \n # 清理特征\n del X_te\n gc.collect()\n \n if not pred.empty and pred.shape[1] > 0:\n # 使用改进的多类预测\n submission_part = predict_multiclass_improved(pred, meta_te, thresholds)\n if len(submission_part) > 0:\n video_submissions.append(submission_part)\n print(f\"\\t 生成 {len(submission_part)} 个提交片段\")\n else:\n print(f\"\\t 预测结果未通过阈值过滤\")\n else:\n print(f\"\\t 没有有效的预测结果\")\n \n # 清理预测结果\n del pred\n gc.collect()\n \n except Exception as e:\n print(f\"\\t 处理 {switch_te} 数据失败: {e}\")\n continue\n \n print(f\"\\t视频 {video_id} 处理完成: {len(video_submissions)} 个提交列表, 处理了 {processed_count} 个生成器项目\")\n return video_submissions\n \n except Exception as e:\n print(f\"\\t处理视频 {row['video_id']} 失败: {e}\")\n return []\n \n # 处理所有视频\n submission_list = []\n print(f\"\\t开始处理 {len(test_subset)} 个视频...\")\n \n # 单进程处理（更稳定）\n for idx, row in test_subset.iterrows():\n print(f\"\\n\\t=== 处理第 {idx + 1}/{len(test_subset)} 个视频 ===\")\n result = process_single_video(row)\n if result:\n submission_list.extend(result)\n # 定期清理内存\n if idx % 5 == 0:\n gc.collect()\n \n # 清理模型缓存\n del models_cache\n gc.collect()\n \n print(f\"\\t完成处理，总共生成 {len(submission_list)} 个提交片段\")\n return submission_list\n\ndef predict_multiclass_improved(pred, meta, thresholds, min_duration=1): # 降低最小持续时间\n \"\"\"修复的预测结果生成函数\"\"\"\n if pred.empty:\n return pd.DataFrame()\n \n # 确保meta有正确的索引\n if not all(col in meta.columns for col in ['video_id', 'agent_id', 'target_id', 'video_frame']):\n print(f\"\\t Meta数据缺少必要列: {meta.columns}\")\n return pd.DataFrame()\n \n ama = np.argmax(pred.values, axis=1)\n max_proba = pred.max(axis=1).values\n\n threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n action_thresholds = threshold_array[ama]\n ama = np.where(max_proba >= action_thresholds, ama, -1)\n \n # 添加最小持续时间过滤\n submission_parts = []\n \n if len(ama) == 0:\n return pd.DataFrame()\n \n current_action = ama[0]\n start_frame = meta.video_frame.iloc[0] if hasattr(meta.video_frame, 'iloc') else meta.video_frame.values[0]\n duration = 1\n \n for i in range(1, len(ama)):\n if ama[i] == current_action:\n duration += 1\n else:\n # 只有持续时间超过阈值的动作才保留\n if current_action >= 0 and duration >= min_duration:\n try:\n video_id = meta.video_id.iloc[i-1] if hasattr(meta.video_id, 'iloc') else meta.video_id.values[i-1]\n agent_id = meta.agent_id.iloc[i-1] if hasattr(meta.agent_id, 'iloc') else meta.agent_id.values[i-1]\n target_id = meta.target_id.iloc[i-1] if hasattr(meta.target_id, 'iloc') else meta.target_id.values[i-1]\n stop_frame = meta.video_frame.iloc[i-1] + 1 if hasattr(meta.video_frame, 'iloc') else meta.video_frame.values[i-1] + 1\n \n submission_parts.append({\n 'video_id': video_id,\n 'agent_id': agent_id,\n 'target_id': target_id,\n 'action': pred.columns[current_action],\n 'start_frame': start_frame,\n 'stop_frame': stop_frame\n })\n except Exception as e:\n print(f\"\\t 创建提交片段失败: {e}\")\n continue\n \n current_action = ama[i]\n start_frame = meta.video_frame.iloc[i] if hasattr(meta.video_frame, 'iloc') else meta.video_frame.values[i]\n duration = 1\n \n # 处理最后一个动作段\n if current_action >= 0 and duration >= min_duration:\n try:\n video_id = meta.video_id.iloc[-1] if hasattr(meta.video_id, 'iloc') else meta.video_id.values[-1]\n agent_id = meta.agent_id.iloc[-1] if hasattr(meta.agent_id, 'iloc') else meta.agent_id.values[-1]\n target_id = meta.target_id.iloc[-1] if hasattr(meta.target_id, 'iloc') else meta.target_id.values[-1]\n stop_frame = meta.video_frame.iloc[-1] + 1 if hasattr(meta.video_frame, 'iloc') else meta.video_frame.values[-1] + 1\n \n submission_parts.append({\n 'video_id': video_id,\n 'agent_id': agent_id,\n 'target_id': target_id,\n 'action': pred.columns[current_action],\n 'start_frame': start_frame,\n 'stop_frame': stop_frame\n })\n except Exception as e:\n print(f\"\\t 创建最后一个提交片段失败: {e}\")\n \n return pd.DataFrame(submission_parts)\n\n# 保持兼容性\nsubmit = submit_improved","metadata":{"papermill":{"duration":0.019182,"end_time":"2025-10-25T19:26:58.650047","exception":false,"start_time":"2025-10-25T19:26:58.630865","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.605547Z","iopub.execute_input":"2025-12-11T03:10:21.605811Z","iopub.status.idle":"2025-12-11T03:10:21.636835Z","shell.execute_reply.started":"2025-12-11T03:10:21.605789Z","shell.execute_reply":"2025-12-11T03:10:21.635956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.mode == \"validate\":\n thresholds = {\n \"single\": {},\n \"pair\": {}\n }\nelse:\n # 修复阈值文件路径\n try:\n thresholds_path = f\"{CFG.model_path}/thresholds.pkl\"\n thresholds = joblib.load(thresholds_path)\n print(f\"成功加载阈值文件: {thresholds_path}\")\n except Exception as e:\n print(f\"加载阈值文件失败: {e}\")\n # 尝试其他可能的路径\n try:\n thresholds_path = f\"/kaggle/input/kk-lightgbm/lightgbm/thresholds.pkl\"\n thresholds = joblib.load(thresholds_path)\n print(f\"成功加载阈值文件: {thresholds_path}\")\n except Exception as e2:\n print(f\"再次加载阈值文件失败: {e2}\")\n thresholds = {\n \"single\": {\"default\": 0.5},\n \"pair\": {\"default\": 0.5}\n }","metadata":{"papermill":{"duration":0.023315,"end_time":"2025-10-25T19:26:58.680526","exception":false,"start_time":"2025-10-25T19:26:58.657211","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.637508Z","iopub.execute_input":"2025-12-11T03:10:21.637758Z","iopub.status.idle":"2025-12-11T03:10:21.654891Z","shell.execute_reply.started":"2025-12-11T03:10:21.637739Z","shell.execute_reply":"2025-12-11T03:10:21.654176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import glob\nimport numpy as np\nimport pandas as pd\nimport json\nimport gc\nimport joblib\nfrom tqdm.notebook import tqdm\n\n# ==========================================\n# 核心修复 1: 智能模型匹配函数\n# ==========================================\ndef find_best_model_match(test_config_str, available_train_configs):\n \"\"\"\n 为测试集的身体部位配置寻找最佳匹配的训练集配置。\n \"\"\"\n # 1. 如果完全匹配，直接返回\n if test_config_str in available_train_configs:\n return test_config_str, \"exact\"\n \n # 2. 尝试解析 JSON\n try:\n test_parts = set(json.loads(test_config_str))\n except:\n # 解析失败，返回第一个可用配置作为兜底\n return available_train_configs[0], \"fallback\"\n\n best_config = available_train_configs[0]\n max_overlap = -1\n \n # 3. 遍历所有训练配置，找交集最大的（重合部位最多的）\n for train_config in available_train_configs:\n try:\n train_parts = set(json.loads(train_config))\n overlap = len(test_parts.intersection(train_parts))\n \n # 如果重合度更高，或者重合度一样但训练配置部位更少（更精简），则更新\n if overlap > max_overlap:\n max_overlap = overlap\n best_config = train_config\n except:\n continue\n \n return best_config, \"closest\"\n\n# ==========================================\n# 核心修复 2: 串行推理函数 (无多进程)\n# ==========================================\ndef submit_improved_serial(test_config_str, model_config_str, section_id, thresholds):\n \"\"\"\n 改进的提交函数（单线程版）：\n 1. test_config_str: 用于从 test dataframe 中筛选数据\n 2. model_config_str: 用于决定特征工程生成哪些列（必须与模型输入对齐）\n 3. section_id: 用于找到磁盘上的模型文件\n \"\"\"\n \n # --- 1. 准备模型需要的身体部位列表 ---\n model_body_parts = json.loads(model_config_str)\n # 如果训练时过滤了某些部位，这里也要保持一致\n if len(model_body_parts) > 5:\n model_body_parts = [b for b in model_body_parts if b not in drop_body_parts]\n \n # --- 2. 筛选当前批次的测试数据 ---\n # 这里我们只处理属于当前 test_config_str 的视频\n test_subset = test[test.body_parts_tracked == test_config_str]\n \n if len(test_subset) == 0:\n return []\n\n print(f\" 测试配置: {test_config_str}\")\n print(f\" 匹配模型: {model_config_str} (Section {section_id})\")\n print(f\" 视频数量: {len(test_subset)}\")\n\n # --- 3. 预加载 FPS 表 ---\n fps_lookup = (\n test_subset[['video_id', 'frames_per_second']]\n .drop_duplicates('video_id')\n .set_index('video_id')['frames_per_second']\n .to_dict()\n )\n\n # --- 4. 预加载模型到内存 ---\n # 先找出这批数据里包含了哪些动作标签（为了只加载需要的模型）\n all_actions = set()\n for behaviors in test_subset['behaviors_labeled']:\n if isinstance(behaviors, str):\n try:\n b_list = json.loads(behaviors)\n for b in b_list:\n act = b.split(',')[-1] if ',' in b else b\n all_actions.add(act)\n except:\n pass\n \n models_cache = {}\n loaded_count = 0\n for action in all_actions:\n # 尝试多个路径查找模型，增加代码鲁棒性\n paths = [\n f\"{CFG.model_path}/{section_id}/{action}/lgb_trainer.pkl\",\n f\"/kaggle/input/kk-lightgbm/lightgbm/{section_id}/{action}/lgb_trainer.pkl\", # 备用路径\n f\"/kaggle/input/kk-lightgbm/{section_id}/{action}/lgb_trainer.pkl\" # 备用路径\n ]\n \n for p in paths:\n if glob.glob(p):\n try:\n models_cache[action] = joblib.load(p)\n loaded_count += 1\n break \n except:\n continue\n \n print(f\" 已加载模型动作数: {loaded_count}\")\n if loaded_count == 0:\n print(\" 警告: 未加载到任何模型，跳过此批次\")\n return []\n\n submission_list = []\n \n # 确定要处理的模式 (Single/Pair)\n modes = []\n # 只要模型加载成功，就尝试运行 single 和 pair，除非你能确定不需要\n modes = ['single', 'pair']\n\n # --- 5. 单线程循环处理视频 ---\n for idx, row in tqdm(test_subset.iterrows(), total=len(test_subset), desc=f\"Processing Sec {section_id}\"):\n try:\n video_id = row['video_id']\n video_data = pd.DataFrame([row])\n \n for switch_tr in modes:\n # 获取当前模式的阈值\n current_thresholds = thresholds.get(switch_tr, {}).get(f\"{section_id}\", {})\n \n # 生成数据\n # 注意：generate_mouse_data 会读取磁盘上的 .parquet 文件\n generator = generate_mouse_data(\n video_data,\n 'test',\n traintest_directory=CFG.test_tracking_path,\n generate_single=(switch_tr == 'single'),\n generate_pair=(switch_tr == 'pair')\n )\n \n for switch_te, data_te, meta_te, actions_te in generator:\n try:\n if 'video_frame' not in meta_te.columns:\n continue\n\n fps_i = _fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n \n # 核心点：使用 [matched model] 的配置来生成特征\n # 这样即使 test 数据多了一些部位，也会被这里忽略，从而匹配模型输入\n if switch_te == 'single':\n X_te = transform_single(data_te, model_body_parts, fps_i).astype(np.float32)\n else:\n X_te = transform_pair(data_te, model_body_parts, fps_i).astype(np.float32)\n \n del data_te\n \n if X_te.shape[0] == 0:\n continue\n \n pred = pd.DataFrame(index=meta_te.video_frame)\n has_prediction = False\n \n # 推理\n for action in actions_te:\n if action in models_cache:\n try:\n models = models_cache[action]\n probas = []\n # 多折模型取平均\n for model in models:\n prob = model.predict_proba(X_te)[:, 1]\n probas.append(prob)\n \n if probas:\n mean_prob = np.mean(probas, axis=0)\n pred[action] = mean_prob\n has_prediction = True\n except:\n continue\n \n del X_te\n \n # 生成结果\n if has_prediction:\n # 兼容性检查：使用 improved 版本或原版本\n if 'predict_multiclass_improved' in globals():\n sub_part = predict_multiclass_improved(pred, meta_te, current_thresholds, min_duration=1)\n else:\n sub_part = predict_multiclass(pred, meta_te, current_thresholds)\n \n if len(sub_part) > 0:\n submission_list.append(sub_part)\n \n del pred\n gc.collect()\n\n except Exception as e:\n print(f\" Error in generator loop: {e}\")\n continue\n \n except Exception as e:\n print(f\" Error processing video {row.get('video_id', 'unknown')}: {e}\")\n continue\n\n # 清理内存\n del models_cache, fps_lookup\n gc.collect()\n \n return submission_list\n\n\n# ==========================================\n# 主执行逻辑 (替换原本的 for 循环)\n# ==========================================\n\n# 1. 加载阈值\nif CFG.mode == \"validate\":\n thresholds = {\"single\": {}, \"pair\": {}}\nelse:\n try:\n pth = f\"{CFG.model_path}/thresholds.pkl\"\n # 备用路径检查\n if not os.path.exists(pth): \n pth = \"/kaggle/input/kk-lightgbm/lightgbm/thresholds.pkl\"\n \n thresholds = joblib.load(pth)\n print(\"Thresholds loaded successfully.\")\n except:\n thresholds = {\"single\": {\"default\": 0.5}, \"pair\": {\"default\": 0.5}}\n print(\"Using default thresholds.\")\n\nsubmission_list = []\n\n# 2. 获取测试集和训练集的配置列表\ntest_unique_configs = test['body_parts_tracked'].unique()\navailable_train_configs = body_parts_tracked_list # 这个变量来自你代码前面的部分\n\nprint(f\"Total unique configs in Test set: {len(test_unique_configs)}\")\n\n# 3. 核心循环：遍历测试集配置\nfor test_config_str in test_unique_configs:\n print(f\"\\n=== Processing Test Config Group ===\")\n \n # 寻找最佳匹配的训练模型\n best_train_config, match_type = find_best_model_match(test_config_str, available_train_configs)\n \n # 获取该训练配置对应的 section ID (因为模型是按 section 文件夹存储的)\n try:\n section_id = available_train_configs.index(best_train_config)\n except ValueError:\n section_id = 1 # 默认值\n \n print(f\"Match Type: {match_type} | Mapped Test Config -> Train Section {section_id}\")\n\n # 调用串行推理\n subs = submit_improved_serial(\n test_config_str=test_config_str,\n model_config_str=best_train_config,\n section_id=section_id,\n thresholds=thresholds\n )\n \n if subs:\n submission_list.extend(subs)\n\n# ==========================================\n# 后处理与保存\n# ==========================================\nif len(submission_list) > 0:\n submission = pd.concat(submission_list)\nelse:\n print(\"Warning: No predictions generated. Creating dummy submission.\")\n submission = pd.DataFrame({\n 'video_id': [test.video_id.iloc[0]],\n 'agent_id': ['mouse1'],\n 'target_id': ['self'], \n 'action': ['other'],\n 'start_frame': [0],\n 'stop_frame': [1]\n })\n\n# Robustify (修复帧重叠等问题)\nsubmission_robust = robustify(submission, test, 'test')\nsubmission_robust.index.name = 'row_id'\n\n# Save\nsubmission_robust.to_csv('submission.csv', index=True)\nprint(f\"Submission saved. Shape: {submission_robust.shape}\")\nprint(submission_robust.head())","metadata":{"papermill":{"duration":1475.342555,"end_time":"2025-10-25T19:51:34.03101","exception":false,"start_time":"2025-10-25T19:26:58.688455","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:10:21.655878Z","iopub.execute_input":"2025-12-11T03:10:21.656216Z","iopub.status.idle":"2025-12-11T03:11:13.750602Z","shell.execute_reply.started":"2025-12-11T03:10:21.656184Z","shell.execute_reply":"2025-12-11T03:11:13.749732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.mode == 'validate': \n submission = pd.concat(submission_list)\n submission_robust = robustify(submission, train, 'train')\n print(f\"Competition metric: {score(solution, submission_robust, ''):.4f}\")\n\n # 修改：根据新的f1_list格式调整DataFrame创建\n # 新的f1_list格式可能是 (body_parts_tracked_str, action, f1, threshold)\n if f1_list and len(f1_list[0]) == 4: # 四元组格式\n f1_df = pd.DataFrame(f1_list, columns=['body_parts_tracked_str', 'action', 'binary F1 score', 'threshold'])\n else: # 三元组格式（兼容旧版本）\n f1_df = pd.DataFrame(f1_list, columns=['body_parts_tracked_str', 'action', 'binary F1 score'])\n \n print(f\"Mean F1: {f1_df['binary F1 score'].mean():.4f}\")\n \n # 修改：确保使用正确的模型名称路径\n os.makedirs(CFG.model_name, exist_ok=True) # 确保目录存在\n joblib.dump(thresholds, f\"{CFG.model_name}/thresholds.pkl\")\n joblib.dump(f1_df, f\"{CFG.model_name}/scores.pkl\")\n print(f\"模型和阈值已保存到 {CFG.model_name}/ 目录\")\n\nif CFG.mode == 'submit':\n if len(submission_list) > 0:\n submission = pd.concat(submission_list)\n else:\n # 提供默认值以防万一\n submission = pd.DataFrame({\n 'video_id': [438887472],\n 'agent_id': ['mouse1'],\n 'target_id': ['self'], \n 'action': ['rear'],\n 'start_frame': [278],\n 'stop_frame': [500]\n })\n \n submission_robust = robustify(submission, test, 'test')\n submission_robust.index.name = 'row_id'\n \n # 保存提交文件\n submission_robust.to_csv('submission.csv', index=True)\n print(f\"提交文件已保存: submission.csv\")\n print(f\"提交文件形状: {submission_robust.shape}\")\n print(\"前几行预览:\")\n print(submission_robust.head())","metadata":{"papermill":{"duration":0.017663,"end_time":"2025-10-25T19:51:34.056443","exception":false,"start_time":"2025-10-25T19:51:34.03878","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T03:11:13.751506Z","iopub.execute_input":"2025-12-11T03:11:13.751791Z","iopub.status.idle":"2025-12-11T03:11:13.888383Z","shell.execute_reply.started":"2025-12-11T03:11:13.751767Z","shell.execute_reply":"2025-12-11T03:11:13.887449Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.009079,"end_time":"2025-10-25T19:51:34.072884","exception":false,"start_time":"2025-10-25T19:51:34.063805","status":"completed"},"tags":[]}},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.154772,"end_time":"2025-10-25T19:51:34.236441","exception":false,"start_time":"2025-10-25T19:51:34.081669","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null}]}