{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13835641,"sourceType":"datasetVersion","datasetId":8811605}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Imports and configs","metadata":{}},{"cell_type":"code","source":"\"\"\"F Beta customized for the data format of the MABe challenge.\"\"\"\n\nimport json\n\nfrom collections import defaultdict\n\nimport pandas as pd\nimport polars as pl\n\n\nclass HostVisibleError(Exception):\n    pass\n\n\ndef single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n    label_frames: defaultdict[str, set[int]] = defaultdict(set)\n    prediction_frames: defaultdict[str, set[int]] = defaultdict(set)\n\n    for row in lab_solution.to_dicts():\n        label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n    for video in lab_solution['video_id'].unique():\n        active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first()  # ty: ignore\n        active_labels: set[str] = set(json.loads(active_labels))\n        predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set)\n\n        for row in lab_submission.filter(pl.col('video_id') == video).to_dicts():\n            # Since the labels are sparse, we can't evaluate prediction keys not in the active labels.\n            if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n                continue\n\n            new_frames = set(range(row['start_frame'], row['stop_frame']))\n            # Ignore truly redundant predictions.\n            new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n            prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n            if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n                # A single agent can have multiple targets per frame (ex: evading all other mice) but only one action per target per frame.\n                raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n            prediction_frames[row['prediction_key']].update(new_frames)\n            predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n    tps = defaultdict(int)\n    fns = defaultdict(int)\n    fps = defaultdict(int)\n    for key, pred_frames in prediction_frames.items():\n        action = key.split('_')[-1]\n        matched_label_frames = label_frames[key]\n        tps[action] += len(pred_frames.intersection(matched_label_frames))\n        fns[action] += len(matched_label_frames.difference(pred_frames))\n        fps[action] += len(pred_frames.difference(matched_label_frames))\n\n    distinct_actions = set()\n    for key, frames in label_frames.items():\n        action = key.split('_')[-1]\n        distinct_actions.add(action)\n        if key not in prediction_frames:\n            fns[action] += len(frames)\n\n    action_f1s = []\n    for action in distinct_actions:\n        if tps[action] + fns[action] + fps[action] == 0:\n            action_f1s.append(0)\n        else:\n            action_f1s.append((1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action]))\n    return sum(action_f1s) / len(action_f1s)\n\n\ndef mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n    \"\"\"\n    Doctests:\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    1.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 0, 'stop_frame': 10}, # Wrong action\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    0.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n    ... ])\n    >>> \"%.12f\" % mouse_fbeta(solution, submission)\n    '0.500000000000'\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 345, 'agent_id': 1, 'target_id': 2, 'action': 'mount', 'start_frame': 15, 'stop_frame': 24, 'lab_id': 2, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 123, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 9},\n    ... ])\n    >>> \"%.12f\" % mouse_fbeta(solution, submission)\n    '0.250000000000'\n\n    >>> # Overlapping solution events, one prediction matching both.\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 10, 'stop_frame': 20, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 20},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    1.0\n\n    >>> solution = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 10, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 30, 'stop_frame': 40, 'lab_id': 1, 'behaviors_labeled': '[\"1,2,attack\"]'},\n    ... ])\n    >>> submission = pd.DataFrame([\n    ...     {'video_id': 1, 'agent_id': 1, 'target_id': 2, 'action': 'attack', 'start_frame': 0, 'stop_frame': 40},\n    ... ])\n    >>> mouse_fbeta(solution, submission)\n    0.6666666666666666\n    \"\"\"\n    if len(solution) == 0 or len(submission) == 0:\n        raise ValueError('Missing solution or submission data')\n\n    expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n    for col in expected_cols:\n        if col not in solution.columns:\n            raise ValueError(f'Solution is missing column {col}')\n        if col not in submission.columns:\n            raise ValueError(f'Submission is missing column {col}')\n\n    solution: pl.DataFrame = pl.DataFrame(solution)\n    submission: pl.DataFrame = pl.DataFrame(submission)\n    assert (solution['start_frame'] <= solution['stop_frame']).all()\n    assert (submission['start_frame'] <= submission['stop_frame']).all()\n    solution_videos = set(solution['video_id'].unique())\n    # Need to align based on video IDs as we can't rely on the row IDs for handling public/private splits.\n    submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n    solution = solution.with_columns(\n        pl.concat_str(\n            [\n                pl.col('video_id').cast(pl.Utf8),\n                pl.col('agent_id').cast(pl.Utf8),\n                pl.col('target_id').cast(pl.Utf8),\n                pl.col('action'),\n            ],\n            separator='_',\n        ).alias('label_key'),\n    )\n    submission = submission.with_columns(\n        pl.concat_str(\n            [\n                pl.col('video_id').cast(pl.Utf8),\n                pl.col('agent_id').cast(pl.Utf8),\n                pl.col('target_id').cast(pl.Utf8),\n                pl.col('action'),\n            ],\n            separator='_',\n        ).alias('prediction_key'),\n    )\n\n    lab_scores = []\n    for lab in solution['lab_id'].unique():\n        lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n        lab_videos = set(lab_solution['video_id'].unique())\n        lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n        lab_scores.append(single_lab_f1(lab_solution, lab_submission, beta=beta))\n\n    return sum(lab_scores) / len(lab_scores)\n\n\ndef score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n    \"\"\"\n    F1 score for the MABe Challenge\n    \"\"\"\n    solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n    submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n    return mouse_fbeta(solution, submission, beta=beta)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:07.152505Z","iopub.execute_input":"2025-11-23T14:08:07.152761Z","iopub.status.idle":"2025-11-23T14:08:08.173120Z","shell.execute_reply.started":"2025-11-23T14:08:07.152740Z","shell.execute_reply":"2025-11-23T14:08:08.172401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import f1_score\nfrom sklearn.base import clone\nfrom xgboost import XGBClassifier\nfrom tqdm.notebook import tqdm\nfrom catboost import CatBoostClassifier\nimport joblib\nimport os\nimport numpy as np\nimport itertools\nimport warnings\nimport optuna\nimport joblib\nimport glob\nimport gc\n\noptuna.logging.set_verbosity(optuna.logging.WARNING)\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:08.174603Z","iopub.execute_input":"2025-11-23T14:08:08.174982Z","iopub.status.idle":"2025-11-23T14:08:09.703148Z","shell.execute_reply.started":"2025-11-23T14:08:08.174958Z","shell.execute_reply":"2025-11-23T14:08:09.702432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    # ===============================\n    # 数据路径（固定）\n    # ===============================\n    train_path = \"/kaggle/input/MABe-mouse-behavior-detection/train.csv\"\n    test_path = \"/kaggle/input/MABe-mouse-behavior-detection/test.csv\"\n    train_annotation_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n    train_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n    test_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\"\n\n    # ===============================\n    # 模型保存路径\n    # ===============================\n    # 模型最终会保存在：\n    # /kaggle/working/models/xgb_cat/{section}/{action}/(xgb_i.pkl, cat_i.pkl)\n    model_path = \"/kaggle/input/mabe-xgb-catb/models\"\n    model_name = \"xgb_cat\"\n\n    # ===============================\n    # 训练 / 提交流程\n    # ===============================\n    # 可选: \"validate\" / \"submit\"\n    mode = \"submit\"\n\n    # ===============================\n    # CV 设置\n    # ===============================\n    n_splits = 3\n    cv = StratifiedGroupKFold(n_splits)\n    random_state = 42\n    # -----------------------\n    # XGBoost 配置（双 T4 折级轮训）\n    # -----------------------\n    xgb_params = dict(\n        verbosity=0,  # 关闭冗余输出\n        random_state=random_state,\n        n_estimators=200,  # 迭代次数（可根据速度调整为 200）\n        learning_rate=0.08,  # 学习率（配合早停使用）\n        max_depth=6,  # 树深度（平衡拟合能力与过拟合）\n        min_child_weight=5,  # 最小子节点权重（抑制过拟合）\n        subsample=0.8,  # 样本采样比例（减少计算量）\n        colsample_bytree=0.8,  # 特征采样比例（减少过拟合）\n        tree_method=\"gpu_hist\",  # 强制 GPU 训练（T4 优化）\n        gpu_id=0,  # 单卡占位（训练时按折切换 0/1）\n        predictor=\"gpu_predictor\",  # GPU 预测（提速）\n        eval_metric=\"logloss\",  # 评估指标（与 CatBoost 一致）\n        early_stopping_rounds=30,  # 早停（避免无效迭代）\n    )\n    \n    # -----------------------\n    # CatBoost 配置（修复 GPU RSM 报错 + 双 T4 并行）\n    # -----------------------\n    cat_params = dict(\n        loss_function=\"Logloss\",  # 损失函数（二分类）\n        eval_metric=\"Logloss\",  # 评估指标（与 XGBoost 一致）\n        iterations=250,  # 迭代次数（可调整为 250 提速）\n        depth=6,  # 树深度（与 XGBoost 保持一致）\n        learning_rate=0.05,  # 学习率（配合早停使用）\n        random_seed=random_state,\n        verbose=False,  # 关闭冗余输出\n        task_type=\"GPU\",  # 强制 GPU 训练\n        devices=\"0,1\",  # 双 T4 GPU（字符串格式，兼容所有版本）\n        gpu_ram_part=0.7,  # 每卡显存限制（16GB T4 预留 30% 显存）\n        bootstrap_type=\"MVS\",  # 采样方式（支持 subsample，性能最优）\n        subsample=0.8,  # 样本采样比例（减少计算量）\n        # colsample_bylevel=0.8,  # 关键删除：GPU 模式二分类不支持该参数\n        early_stopping_rounds=30,  # 早停（避免无效迭代）\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.704049Z","iopub.execute_input":"2025-11-23T14:08:09.704461Z","iopub.status.idle":"2025-11-23T14:08:09.713522Z","shell.execute_reply.started":"2025-11-23T14:08:09.704435Z","shell.execute_reply":"2025-11-23T14:08:09.712724Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(CFG.train_path)\ntrain['n_mice'] = 4 - train[['mouse1_strain', 'mouse2_strain', 'mouse3_strain', 'mouse4_strain']].isna().sum(axis=1)\ntrain_without_mabe22 = train.query(\"~lab_id.str.startswith('MABe22_')\")\n\ntest = pd.read_csv(CFG.test_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.714417Z","iopub.execute_input":"2025-11-23T14:08:09.714726Z","iopub.status.idle":"2025-11-23T14:08:09.911281Z","shell.execute_reply.started":"2025-11-23T14:08:09.714698Z","shell.execute_reply":"2025-11-23T14:08:09.910580Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"body_parts_tracked_list = list(np.unique(train.body_parts_tracked))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.913322Z","iopub.execute_input":"2025-11-23T14:08:09.913889Z","iopub.status.idle":"2025-11-23T14:08:09.922596Z","shell.execute_reply.started":"2025-11-23T14:08:09.913861Z","shell.execute_reply":"2025-11-23T14:08:09.921692Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Creating solution data","metadata":{}},{"cell_type":"code","source":"def create_solution_df(dataset):\n    solution = []\n    for _, row in tqdm(dataset.iterrows(), total=len(dataset)):\n    \n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'): \n            continue\n        \n        video_id = row['video_id']\n        path = f\"{CFG.train_annotation_path}/{lab_id}/{video_id}.parquet\"\n        try:\n            annot = pd.read_parquet(path)\n        except FileNotFoundError:\n            continue\n    \n        annot['lab_id'] = lab_id\n        annot['video_id'] = video_id\n        annot['behaviors_labeled'] = row['behaviors_labeled']\n        annot['target_id'] = np.where(annot.target_id != annot.agent_id, annot['target_id'].apply(lambda s: f\"mouse{s}\"), 'self')\n        annot['agent_id'] = annot['agent_id'].apply(lambda s: f\"mouse{s}\")\n        solution.append(annot)\n    \n    solution = pd.concat(solution)\n    \n    return solution\n\nif CFG.mode == 'validate':\n    solution = create_solution_df(train_without_mabe22)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.923345Z","iopub.execute_input":"2025-11-23T14:08:09.923615Z","iopub.status.idle":"2025-11-23T14:08:09.939028Z","shell.execute_reply.started":"2025-11-23T14:08:09.923591Z","shell.execute_reply":"2025-11-23T14:08:09.938210Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Data generator","metadata":{}},{"cell_type":"code","source":"import json\nimport itertools\nimport pandas as pd\nimport numpy as np\nimport gc\nfrom tqdm import tqdm\n\ndrop_body_parts = [\n    'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n    'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright', \n    'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint'\n]\n\n\n\ndef generate_mouse_data(dataset, traintest, traintest_directory=None, generate_single=True, generate_pair=True):\n    import json\n    import itertools\n    import pandas as pd\n    import numpy as np\n    import gc\n    import os\n    \n    # ---------------------- 设置数据集路径 ----------------------\n    if traintest_directory is None:\n        if traintest == 'test':\n            traintest_directory = \"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\"\n        else:\n            traintest_directory = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n        print(f\"使用数据集路径：{traintest_directory}\")\n    \n    # ---------------------- 过滤有效数据 ----------------------\n    mask_lab = dataset.lab_id.str.startswith('MABe22', na=True)\n    mask_behavior = dataset.behaviors_labeled.apply(\n        lambda x: isinstance(x, str) and x.strip() != ''\n    )\n    valid_dataset = dataset[~mask_lab & mask_behavior].copy()\n    \n    if len(valid_dataset) == 0:\n        print(f\"\\t⚠️  无有效数据可处理，返回空迭代器\")\n        return\n    \n    # ---------------------- 迭代处理每个视频 ----------------------\n    for _, row in valid_dataset.iterrows():\n        lab_id = row.lab_id\n        video_id = str(int(row.video_id))\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        \n        print(f\"\\t🔍 正在处理视频：video_id={video_id}, lab_id={lab_id}, 路径={path}\")\n        \n        if not os.path.exists(path):\n            print(f\"\\t⚠️  跳过不存在的文件：{path}\")\n            continue\n        \n        # 读取追踪文件\n        try:\n            vid = pd.read_parquet(path)\n            print(f\"\\t✅ 成功读取追踪文件，数据形状：{vid.shape}，列名：{list(vid.columns)}\")\n            print(f\"\\t✅ 追踪文件中 mouse_id 唯一值：{vid['mouse_id'].unique()}\")\n            print(f\"\\t✅ 追踪文件中 bodypart 唯一值：{list(vid['bodypart'].unique())[:10]}...（共{len(vid['bodypart'].unique())}个）\")\n        except Exception as e:\n            print(f\"\\t⚠️  读取文件 {path} 失败：{str(e)}\")\n            continue\n        \n        # 过滤不需要的身体部位（依赖全局 drop_body_parts，若未定义则跳过过滤）\n        if 'drop_body_parts' in globals() and len(np.unique(vid.bodypart)) > 5:\n            before_drop = len(vid['bodypart'].unique())\n            vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n            after_drop = len(vid['bodypart'].unique())\n            print(f\"\\t✅ 身体部位过滤：从 {before_drop} 个减少到 {after_drop} 个，保留：{list(vid['bodypart'].unique())[:10]}...\")\n        \n        # 透视表整理数据\n        try:\n            pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n            pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n            pvid /= row.pix_per_cm_approx\n            print(f\"\\t✅ 透视表整理完成，形状：{pvid.shape}，列名层级：{pvid.columns.names}\")\n            print(f\"\\t✅ 透视表中有效小鼠ID：{list(pvid.columns.get_level_values('mouse_id').unique())}\")\n        except Exception as e:\n            print(f\"\\t⚠️  整理文件 {path} 数据失败：{str(e)}\")\n            del vid\n            gc.collect()\n            continue\n        \n        # 检查透视表是否为空\n        if pvid.empty or len(pvid.columns) == 0:\n            print(f\"\\t⚠️  透视表为空或无有效列，跳过该视频\")\n            del vid, pvid\n            gc.collect()\n            continue\n        \n        del vid\n        gc.collect()\n        \n        # ---------------------- 解析行为标签（修复：支持 mouseX,self,action 格式）----------------------\n        try:\n            vid_behaviors = json.loads(row.behaviors_labeled)\n            vid_behaviors = sorted(list({b.replace(\"'\", \"\").strip() for b in vid_behaviors}))\n            print(f\"\\t✅ 解析行为标签：原始行为列表={vid_behaviors[:5]}...（共{len(vid_behaviors)}个）\")\n            \n            if traintest == 'test':\n                # 测试集：标签是 \"mouseA,mouseB,action\" 或 \"mouseA,self,action\" 格式，直接拆分\n                vid_behaviors_df = []\n                for behavior_str in vid_behaviors:\n                    parts = behavior_str.split(',')\n                    if len(parts) == 3:\n                        agent_str = parts[0].strip()  # 比如 \"mouse1\"\n                        target_str = parts[1].strip()  # 比如 \"mouse2\" 或 \"self\"\n                        action = parts[2].strip()      # 比如 \"approach\" 或 \"rear\"\n                        \n                        # 验证格式：agent 必须是 \"mouseX\"，target 可以是 \"mouseX\" 或 \"self\"\n                        if agent_str.startswith('mouse') and (target_str.startswith('mouse') or target_str == 'self'):\n                            vid_behaviors_df.append([agent_str, target_str, action])\n                            # 只打印前10个行为，避免日志过长\n                            if len(vid_behaviors_df) <= 10:\n                                print(f\"\\t\\t🔍 拆分行为：agent={agent_str}, target={target_str}, action={action}\")\n                        else:\n                            print(f\"\\t\\t⚠️  无效格式：{behavior_str}（agent需为mouseX，target需为mouseX或self），跳过\")\n                    else:\n                        print(f\"\\t\\t⚠️  无效格式：{behavior_str}（拆分后不是3部分），跳过\")\n                \n                # 转换为DataFrame并过滤空数据\n                vid_behaviors = pd.DataFrame(vid_behaviors_df, columns=['agent', 'target', 'action'])\n                vid_behaviors = vid_behaviors.dropna()\n            else:\n                # 训练集：原有逻辑（三维列表）\n                vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n            \n            print(f\"\\t✅ 最终行为数据框：\\n{vid_behaviors.head()}\")\n            if vid_behaviors.empty:\n                print(f\"\\t⚠️  无有效行为数据，跳过该视频\")\n                del pvid, vid_behaviors\n                gc.collect()\n                continue\n            \n            # 验证行为数据框中的小鼠ID是否存在于追踪文件中\n            valid_mouse_ids = [f\"mouse{mid}\" for mid in pvid.columns.get_level_values('mouse_id').unique()]\n            invalid_agents = [a for a in vid_behaviors['agent'].unique() if a not in valid_mouse_ids]\n            invalid_targets = [t for t in vid_behaviors['target'].unique() if t not in valid_mouse_ids and t != 'self']\n            \n            if invalid_agents:\n                print(f\"\\t⚠️  无效agent：{invalid_agents}（追踪文件中只有 {valid_mouse_ids}），会过滤掉这些行为\")\n            if invalid_targets:\n                print(f\"\\t⚠️  无效target：{invalid_targets}（追踪文件中只有 {valid_mouse_ids}），会过滤掉这些行为\")\n            \n            # 过滤掉无效小鼠ID的行为\n            vid_behaviors = vid_behaviors[\n                (vid_behaviors['agent'].isin(valid_mouse_ids)) & \n                ((vid_behaviors['target'].isin(valid_mouse_ids)) | (vid_behaviors['target'] == 'self'))\n            ].reset_index(drop=True)\n            print(f\"\\t✅ 过滤后有效行为数：{len(vid_behaviors)}\")\n\n        except Exception as e:\n            print(f\"\\t⚠️  解析行为标签失败：{str(e)}\")\n            print(f\"\\t⚠️  行为标签原始数据：{str(row.behaviors_labeled)[:100]}...\")\n            del pvid\n            gc.collect()\n            continue\n        \n        # ---------------------- 训练模式：读取标注文件 ----------------------\n        annot = None\n        if traintest == 'train':\n            try:\n                annot_path = path.replace('train_tracking', 'train_annotation')\n                if os.path.exists(annot_path):\n                    annot = pd.read_parquet(annot_path)\n                else:\n                    print(f\"\\t⚠️  训练标注文件不存在：{annot_path}，跳过该视频\")\n                    del pvid, vid_behaviors\n                    gc.collect()\n                    continue\n            except Exception as e:\n                print(f\"\\t⚠️  读取标注文件失败：{str(e)}\")\n                del pvid, vid_behaviors\n                gc.collect()\n                continue\n        \n        # ---------------------- 生成单鼠行为数据 ----------------------\n        single_generated = 0\n        if generate_single:\n            vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n            print(f\"\\t🔍 单鼠行为子集形状：{vid_behaviors_subset.shape}\")\n            if len(vid_behaviors_subset) > 0:\n                for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n                    try:\n                        mouse_id = int(mouse_id_str[-1])\n                        vid_agent_actions = np.unique(vid_behaviors_subset.query(\"agent == @mouse_id_str\").action)\n                        print(f\"\\t🔍 单鼠 {mouse_id_str} 对应的行为：{vid_agent_actions}\")\n                        \n                        # 检查小鼠数据是否存在\n                        if mouse_id not in pvid.columns.get_level_values('mouse_id'):\n                            print(f\"\\t⚠️  单鼠 {mouse_id_str} 在透视表中无数据，跳过\")\n                            continue\n                        \n                        single_mouse = pvid.loc[:, mouse_id]\n                        assert len(single_mouse) == len(pvid), \"单鼠数据长度不匹配\"\n                        \n                        single_mouse_meta = pd.DataFrame({\n                            'video_id': video_id,\n                            'agent_id': mouse_id_str,\n                            'target_id': 'self',\n                            'video_frame': single_mouse.index\n                        })\n                        \n                        if traintest == 'train':\n                            single_mouse_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=single_mouse.index)\n                            annot_subset = annot.query(\"(agent_id == @mouse_id) & (target_id == @mouse_id)\")\n                            for _, annot_row in annot_subset.iterrows():\n                                mask = (single_mouse_label.index >= annot_row['start_frame']) & (single_mouse_label.index <= annot_row['stop_frame'])\n                                single_mouse_label.loc[mask, annot_row.action] = 1.0\n                            yield 'single', single_mouse, single_mouse_meta, single_mouse_label\n                        else:\n                            yield 'single', single_mouse, single_mouse_meta, vid_agent_actions\n                        single_generated += 1\n                    except KeyError as e:\n                        print(f\"\\t⚠️  单鼠行为处理失败（KeyError）：{str(e)}\")\n                        continue\n                    except Exception as e:\n                        print(f\"\\t⚠️  单鼠行为处理失败：{str(e)}\")\n                        continue\n        print(f\"\\t✅ 单鼠数据生成总数：{single_generated}\")\n        \n        # ---------------------- 生成双鼠行为数据 ----------------------\n        pair_generated = 0\n        if generate_pair:\n            vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n            print(f\"\\t🔍 双鼠行为子集形状：{vid_behaviors_subset.shape}\")\n            if len(vid_behaviors_subset) > 0:\n                mouse_ids = np.unique(pvid.columns.get_level_values('mouse_id'))\n                print(f\"\\t🔍 可用小鼠ID：{mouse_ids}\")\n                for agent, target in itertools.permutations(mouse_ids, 2):\n                    agent_str = f\"mouse{agent}\"\n                    target_str = f\"mouse{target}\"\n                    vid_agent_actions = np.unique(\n                        vid_behaviors_subset.query(\"(agent == @agent_str) & (target == @target_str)\").action\n                    )\n                    print(f\"\\t🔍 双鼠 {agent_str}→{target_str} 对应的行为：{vid_agent_actions}\")\n                    if len(vid_agent_actions) == 0:\n                        continue\n                    \n                    try:\n                        mouse_pair = pd.concat([pvid[agent], pvid[target]], axis=1, keys=['A', 'B'])\n                        assert len(mouse_pair) == len(pvid), \"双鼠数据长度不匹配\"\n                        \n                        mouse_pair_meta = pd.DataFrame({\n                            'video_id': video_id,\n                            'agent_id': agent_str,\n                            'target_id': target_str,\n                            'video_frame': mouse_pair.index\n                        })\n                        \n                        if traintest == 'train':\n                            mouse_pair_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=mouse_pair.index)\n                            annot_subset = annot.query(\"(agent_id == @agent) & (target_id == @target)\")\n                            for _, annot_row in annot_subset.iterrows():\n                                mask = (mouse_pair_label.index >= annot_row['start_frame']) & (mouse_pair_label.index <= annot_row['stop_frame'])\n                                mouse_pair_label.loc[mask, annot_row.action] = 1.0\n                            yield 'pair', mouse_pair, mouse_pair_meta, mouse_pair_label\n                        else:\n                            yield 'pair', mouse_pair, mouse_pair_meta, vid_agent_actions\n                        pair_generated += 1\n                    except KeyError as e:\n                        print(f\"\\t⚠️  双鼠行为处理失败（KeyError）：{str(e)}\")\n                        continue\n                    except Exception as e:\n                        print(f\"\\t⚠️  双鼠行为处理失败：{str(e)}\")\n                        continue\n        print(f\"\\t✅ 双鼠数据生成总数：{pair_generated}\")\n        \n        # 清理临时数据\n        del pvid, vid_behaviors\n        if annot is not None:\n            del annot\n        gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.939945Z","iopub.execute_input":"2025-11-23T14:08:09.940192Z","iopub.status.idle":"2025-11-23T14:08:09.971259Z","shell.execute_reply.started":"2025-11-23T14:08:09.940175Z","shell.execute_reply":"2025-11-23T14:08:09.970439Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Transforming coordinates","metadata":{}},{"cell_type":"code","source":"def safe_rolling(series, window, func, min_periods=None):\n    if min_periods is None:\n        min_periods = max(1, window // 4)\n    return series.rolling(window, min_periods=min_periods, center=True).apply(func, raw=True)\n\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n    if n_frames_at_30fps == 0:\n        return 0\n    s = 1 if n_frames_at_30fps > 0 else -1\n    mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n    return s * mag\n\ndef _fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n    if 'frames_per_second' in meta_df.columns and pd.notnull(meta_df['frames_per_second']).any():\n        return float(meta_df['frames_per_second'].iloc[0])\n    vid = meta_df['video_id'].iloc[0]\n    return float(fallback_lookup.get(vid, default_fps))\n\ndef add_curvature_features(X, center_x, center_y, fps):\n    vel_x = center_x.diff()\n    vel_y = center_y.diff()\n    acc_x = vel_x.diff()\n    acc_y = vel_y.diff()\n\n    cross_prod = vel_x * acc_y - vel_y * acc_x\n    vel_mag = np.sqrt(vel_x**2 + vel_y**2)\n    curvature = np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n\n    for w in [25, 50, 75]:\n        ws = _scale(w, fps)\n        X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 5)).mean()\n\n    angle = np.arctan2(vel_y, vel_x)\n    angle_change = np.abs(angle.diff())\n    w = 30\n    ws = _scale(w, fps)\n    X[f'turn_rate_{w}'] = angle_change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n\n    return X\n\ndef add_multiscale_features(X, center_x, center_y, fps):\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n\n    scales = [20, 40, 60, 80]\n    for scale in scales:\n        ws = _scale(scale, fps)\n        if len(speed) >= ws:\n            X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n            X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n\n    if len(scales) >= 2 and f'sp_m{scales[0]}' in X.columns and f'sp_m{scales[-1]}' in X.columns:\n        X['sp_ratio'] = X[f'sp_m{scales[0]}'] / (X[f'sp_m{scales[-1]}'] + 1e-6)\n\n    return X\n\ndef add_state_features(X, center_x, center_y, fps):\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n    w_ma = _scale(15, fps)\n    speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n\n    try:\n        bins = [-np.inf, 0.5 * fps, 2.0 * fps, 5.0 * fps, np.inf]\n        speed_states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n\n        for window in [20, 40, 60, 80]:\n            ws = _scale(window, fps)\n            if len(speed_states) >= ws:\n                for state in [0, 1, 2, 3]:\n                    X[f's{state}_{window}'] = (\n                        (speed_states == state).astype(float)\n                        .rolling(ws, min_periods=max(1, ws // 5)).mean()\n                    )\n                state_changes = (speed_states != speed_states.shift(1)).astype(float)\n                X[f'trans_{window}'] = state_changes.rolling(ws, min_periods=max(1, ws // 5)).sum()\n    except Exception:\n        pass\n\n    return X\n\ndef add_longrange_features(X, center_x, center_y, fps):\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(center_x) >= ws:\n            X[f'x_ml{window}'] = center_x.rolling(ws, min_periods=max(5, ws // 6)).mean()\n            X[f'y_ml{window}'] = center_y.rolling(ws, min_periods=max(5, ws // 6)).mean()\n\n    for span in [30, 60, 120]:\n        s = _scale(span, fps)\n        X[f'x_e{span}'] = center_x.ewm(span=s, min_periods=1).mean()\n        X[f'y_e{span}'] = center_y.ewm(span=s, min_periods=1).mean()\n\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)  # cm/s\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(speed) >= ws:\n            X[f'sp_pct{window}'] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n\n    return X\n\ndef add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n    if 'body_center' not in avail_A or 'body_center' not in avail_B:\n        return X\n\n    rel_x = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n    rel_y = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n    rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n    A_vx = mouse_pair['A']['body_center']['x'].diff()\n    A_vy = mouse_pair['A']['body_center']['y'].diff()\n    B_vx = mouse_pair['B']['body_center']['x'].diff()\n    B_vy = mouse_pair['B']['body_center']['y'].diff()\n\n    A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n    B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n    for window in [30, 60]:\n        ws = _scale(window, fps)\n        X[f'A_ld{window}'] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n        X[f'B_ld{window}'] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    approach = -rel_dist.diff()\n    chase = approach * B_lead\n    w = 30\n    ws = _scale(w, fps)\n    X[f'chase_{w}'] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    for window in [60, 120]:\n        ws = _scale(window, fps)\n        A_sp = np.sqrt(A_vx**2 + A_vy**2)\n        B_sp = np.sqrt(B_vx**2 + B_vy**2)\n        X[f'sp_cor{window}'] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n\n    return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:09.972122Z","iopub.execute_input":"2025-11-23T14:08:09.972442Z","iopub.status.idle":"2025-11-23T14:08:10.004762Z","shell.execute_reply.started":"2025-11-23T14:08:09.972422Z","shell.execute_reply":"2025-11-23T14:08:10.004022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def transform_single(single_mouse, body_parts_tracked, fps):\n    available_body_parts = single_mouse.columns.get_level_values(0)\n\n    X = pd.DataFrame({\n        f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n        if p1 in available_body_parts and p2 in available_body_parts\n    })\n    X = X.reindex(columns=[f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(body_parts_tracked, 2)], copy=False)\n\n    if all(p in single_mouse.columns for p in ['ear_left', 'ear_right', 'tail_base']):\n        lag = _scale(10, fps)\n        shifted = single_mouse[['ear_left', 'ear_right', 'tail_base']].shift(lag)\n        speeds = pd.DataFrame({\n            'sp_lf': np.square(single_mouse['ear_left'] - shifted['ear_left']).sum(axis=1, skipna=False),\n            'sp_rt': np.square(single_mouse['ear_right'] - shifted['ear_right']).sum(axis=1, skipna=False),\n            'sp_lf2': np.square(single_mouse['ear_left'] - shifted['tail_base']).sum(axis=1, skipna=False),\n            'sp_rt2': np.square(single_mouse['ear_right'] - shifted['tail_base']).sum(axis=1, skipna=False),\n        })\n        X = pd.concat([X, speeds], axis=1)\n\n    if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n        X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n    if all(p in available_body_parts for p in ['nose', 'body_center', 'tail_base']):\n        v1 = single_mouse['nose'] - single_mouse['body_center']\n        v2 = single_mouse['tail_base'] - single_mouse['body_center']\n        X['body_ang'] = (v1['x'] * v2['x'] + v1['y'] * v2['y']) / (\n            np.sqrt(v1['x']**2 + v1['y']**2) * np.sqrt(v2['x']**2 + v2['y']**2) + 1e-6)\n\n    if 'body_center' in available_body_parts:\n        cx = single_mouse['body_center']['x']\n        cy = single_mouse['body_center']['y']\n\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f'cx_m{w}'] = cx.rolling(ws, **roll).mean()\n            X[f'cy_m{w}'] = cy.rolling(ws, **roll).mean()\n            X[f'cx_s{w}'] = cx.rolling(ws, **roll).std()\n            X[f'cy_s{w}'] = cy.rolling(ws, **roll).std()\n            X[f'x_rng{w}'] = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n            X[f'y_rng{w}'] = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n            X[f'disp{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).sum()**2 +\n                                     cy.diff().rolling(ws, min_periods=1).sum()**2)\n            X[f'act{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).var() +\n                                   cy.diff().rolling(ws, min_periods=1).var())\n\n        X = add_curvature_features(X, cx, cy, fps)\n        X = add_multiscale_features(X, cx, cy, fps)\n        X = add_state_features(X, cx, cy, fps)\n        X = add_longrange_features(X, cx, cy, fps)\n\n    if all(p in available_body_parts for p in ['nose', 'tail_base']):\n        nt_dist = np.sqrt((single_mouse['nose']['x'] - single_mouse['tail_base']['x'])**2 +\n                          (single_mouse['nose']['y'] - single_mouse['tail_base']['y'])**2)\n        for lag in [10, 20, 40]:\n            l = _scale(lag, fps)\n            X[f'nt_lg{lag}'] = nt_dist.shift(l)\n            X[f'nt_df{lag}'] = nt_dist - nt_dist.shift(l)\n\n    if all(p in available_body_parts for p in ['ear_left', 'ear_right']):\n        ear_d = np.sqrt((single_mouse['ear_left']['x'] - single_mouse['ear_right']['x'])**2 +\n                        (single_mouse['ear_left']['y'] - single_mouse['ear_right']['y'])**2)\n        for off in [-30, -20, -10, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f'ear_o{off}'] = ear_d.shift(-o)\n        w = _scale(30, fps)\n        X['ear_con'] = ear_d.rolling(w, min_periods=1, center=True).std() / \\\n                       (ear_d.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n    return X.astype(np.float32, copy=False)\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n    avail_A = mouse_pair['A'].columns.get_level_values(0)\n    avail_B = mouse_pair['B'].columns.get_level_values(0)\n\n    X = pd.DataFrame({\n        f\"12+{p1}+{p2}\": np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n        if p1 in avail_A and p2 in avail_B\n    })\n    X = X.reindex(columns=[f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(body_parts_tracked, repeat=2)], copy=False)\n\n    if ('A', 'ear_left') in mouse_pair.columns and ('B', 'ear_left') in mouse_pair.columns:\n        lag = _scale(10, fps)\n        shA = mouse_pair['A']['ear_left'].shift(lag)\n        shB = mouse_pair['B']['ear_left'].shift(lag)\n        speeds = pd.DataFrame({\n            'sp_A': np.square(mouse_pair['A']['ear_left'] - shA).sum(axis=1, skipna=False),\n            'sp_AB': np.square(mouse_pair['A']['ear_left'] - shB).sum(axis=1, skipna=False),\n            'sp_B': np.square(mouse_pair['B']['ear_left'] - shB).sum(axis=1, skipna=False),\n        })\n        X = pd.concat([X, speeds], axis=1)\n\n    if 'nose+tail_base' in X.columns and 'ear_left+ear_right' in X.columns:\n        X['elong'] = X['nose+tail_base'] / (X['ear_left+ear_right'] + 1e-6)\n\n    if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n        dir_A = mouse_pair['A']['nose'] - mouse_pair['A']['tail_base']\n        dir_B = mouse_pair['B']['nose'] - mouse_pair['B']['tail_base']\n        X['rel_ori'] = (dir_A['x'] * dir_B['x'] + dir_A['y'] * dir_B['y']) / (\n            np.sqrt(dir_A['x']**2 + dir_A['y']**2) * np.sqrt(dir_B['x']**2 + dir_B['y']**2) + 1e-6)\n\n    if all(p in avail_A for p in ['nose']) and all(p in avail_B for p in ['nose']):\n        cur = np.square(mouse_pair['A']['nose'] - mouse_pair['B']['nose']).sum(axis=1, skipna=False)\n        lag = _scale(10, fps)\n        shA_n = mouse_pair['A']['nose'].shift(lag)\n        shB_n = mouse_pair['B']['nose'].shift(lag)\n        past = np.square(shA_n - shB_n).sum(axis=1, skipna=False)\n        X['appr'] = cur - past\n\n    if 'body_center' in avail_A and 'body_center' in avail_B:\n        cd = np.sqrt((mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x'])**2 +\n                     (mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y'])**2)\n        X['v_cls'] = (cd < 5.0).astype(float)\n        X['cls']   = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n        X['med']   = ((cd >= 15.0) & (cd < 30.0)).astype(float)\n        X['far']   = (cd >= 30.0).astype(float)\n\n    if 'body_center' in avail_A and 'body_center' in avail_B:\n        cd_full = np.square(mouse_pair['A']['body_center'] - mouse_pair['B']['body_center']).sum(axis=1, skipna=False)\n\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f'd_m{w}']  = cd_full.rolling(ws, **roll).mean()\n            X[f'd_s{w}']  = cd_full.rolling(ws, **roll).std()\n            X[f'd_mn{w}'] = cd_full.rolling(ws, **roll).min()\n            X[f'd_mx{w}'] = cd_full.rolling(ws, **roll).max()\n\n            d_var = cd_full.rolling(ws, **roll).var()\n            X[f'int{w}'] = 1 / (1 + d_var)\n\n            Axd = mouse_pair['A']['body_center']['x'].diff()\n            Ayd = mouse_pair['A']['body_center']['y'].diff()\n            Bxd = mouse_pair['B']['body_center']['x'].diff()\n            Byd = mouse_pair['B']['body_center']['y'].diff()\n            coord = Axd * Bxd + Ayd * Byd\n            X[f'co_m{w}'] = coord.rolling(ws, **roll).mean()\n            X[f'co_s{w}'] = coord.rolling(ws, **roll).std()\n\n    if 'nose' in avail_A and 'nose' in avail_B:\n        nn = np.sqrt((mouse_pair['A']['nose']['x'] - mouse_pair['B']['nose']['x'])**2 +\n                     (mouse_pair['A']['nose']['y'] - mouse_pair['B']['nose']['y'])**2)\n        for lag in [10, 20, 40]:\n            l = _scale(lag, fps)\n            X[f'nn_lg{lag}']  = nn.shift(l)\n            X[f'nn_ch{lag}']  = nn - nn.shift(l)\n            is_cl = (nn < 10.0).astype(float)\n            X[f'cl_ps{lag}']  = is_cl.rolling(l, min_periods=1).mean()\n\n    if 'body_center' in avail_A and 'body_center' in avail_B:\n        Avx = mouse_pair['A']['body_center']['x'].diff()\n        Avy = mouse_pair['A']['body_center']['y'].diff()\n        Bvx = mouse_pair['B']['body_center']['x'].diff()\n        Bvy = mouse_pair['B']['body_center']['y'].diff()\n        val = (Avx * Bvx + Avy * Bvy) / (np.sqrt(Avx**2 + Avy**2) * np.sqrt(Bvx**2 + Bvy**2) + 1e-6)\n\n        for off in [-30, -20, -10, 0, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f'va_{off}'] = val.shift(-o)\n\n        w = _scale(30, fps)\n        X['int_con'] = cd_full.rolling(w, min_periods=1, center=True).std() / \\\n                       (cd_full.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n        X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n    return X.astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:10.005639Z","iopub.execute_input":"2025-11-23T14:08:10.005935Z","iopub.status.idle":"2025-11-23T14:08:10.043253Z","shell.execute_reply.started":"2025-11-23T14:08:10.005907Z","shell.execute_reply":"2025-11-23T14:08:10.042350Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training, validation and submission","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\n\ndef robustify(submission, dataset, traintest, traintest_directory=None):\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n    \n    old_submission = submission.copy()\n    \n    # --------------------------\n    # 核心修复：将 start_frame/stop_frame 转为整数\n    # --------------------------\n    # 处理字符串转整数（兼容 \"123\" 这类字符串，无效值转为 NaN 后填充为 0）\n    submission['start_frame'] = pd.to_numeric(submission['start_frame'], errors='coerce').fillna(0).astype(int)\n    submission['stop_frame'] = pd.to_numeric(submission['stop_frame'], errors='coerce').fillna(0).astype(int)\n    \n    # 过滤 start >= stop 的无效行（现在是整数比较，无类型错误）\n    submission = submission[submission.start_frame < submission.stop_frame]\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped frames with start >= stop\")\n    \n    old_submission = submission.copy()\n    group_list = []\n    # 按 video_id + agent_id + target_id 分组，避免跨主体的帧冲突\n    for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n        group = group.sort_values('start_frame')  # 按开始帧排序\n        mask = np.ones(len(group), dtype=bool)\n        last_stop_frame = 0\n        for i, (_, row) in enumerate(group.iterrows()):\n            # 现在 start_frame 是整数，可正常比较\n            if row['start_frame'] < last_stop_frame:\n                mask[i] = False  # 过滤重叠的动作（当前开始帧 < 上一个结束帧）\n            else:\n                last_stop_frame = row['stop_frame']  # 更新上一个动作的结束帧\n        group_list.append(group[mask])\n        \n    submission = pd.concat(group_list)\n    \n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped duplicate frames\")\n        \n    s_list = []\n    # 处理无预测结果的视频，填充默认动作帧\n    for idx, row in dataset.iterrows():\n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'):\n            continue\n        \n        video_id = row['video_id']\n        if (submission.video_id == video_id).any():\n            continue  # 已有预测结果，跳过\n        \n        if type(row.behaviors_labeled) != str:\n            continue\n\n        print(f\"Video {video_id} has no predictions.\")\n        \n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n    \n        vid_behaviors = json.loads(row['behaviors_labeled'])\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(',') for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n    \n        start_frame = vid.video_frame.min()\n        stop_frame = vid.video_frame.max() + 1\n    \n        # 按 agent + target 分组，均匀分配动作帧\n        for (agent, target), actions in vid_behaviors.groupby(['agent', 'target']):\n            batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n            for i, (_, action_row) in enumerate(actions.iterrows()):\n                batch_start = start_frame + i * batch_length\n                batch_stop = min(batch_start + batch_length, stop_frame)\n                s_list.append((video_id, agent, target, action_row['action'], batch_start, batch_stop))\n\n    if len(s_list) > 0:\n        # 新增的填充行，start_frame/stop_frame 本身是整数，无需转换\n        submission = pd.concat([\n            submission,\n            pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n        ])\n        print(\"ERROR: Filled empty videos\")\n\n    submission = submission.reset_index(drop=True)\n    \n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:10.044147Z","iopub.execute_input":"2025-11-23T14:08:10.044405Z","iopub.status.idle":"2025-11-23T14:08:10.069779Z","shell.execute_reply.started":"2025-11-23T14:08:10.044383Z","shell.execute_reply":"2025-11-23T14:08:10.068969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_multiclass(pred, meta, thresholds):\n    \"\"\"\n    生成比赛要求的帧区间预测结果（保留原始行为变化检测逻辑）\n    输出列：video_id, agent_id, target_id, action, start_frame, stop_frame\n    \"\"\"\n    import pandas as pd\n    import numpy as np\n    \n    # 1. 取概率最大的行为和对应概率\n    ama = np.argmax(pred.values, axis=1)\n    max_proba = pred.max(axis=1).values\n\n    # 2. 行为阈值匹配\n    threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n    action_thresholds = threshold_array[ama]\n\n    # 3. 过滤概率低于阈值的行为（标记为-1）\n    ama = np.where(max_proba >= action_thresholds, ama, -1)\n    ama = pd.Series(ama, index=meta['video_frame'].values)  # 索引=原始帧号\n    \n    # 4. 检测行为变化点（和上一帧不同即为变化）\n    changes_mask = (ama != ama.shift(1)).values\n    ama_changes = ama[changes_mask]\n    meta_changes = meta[changes_mask].reset_index(drop=True)  # 重置索引避免对齐错误\n    \n    # 5. 过滤无行为的变化点，最后一个变化点设为False（避免索引越界）\n    mask = ama_changes.values >= 0\n    if len(mask) > 0:\n        mask[-1] = False\n    \n    # 6. 生成核心预测结果（比赛要求的5列）\n    submission_part = pd.DataFrame()\n    if np.any(mask):\n        submission_part = pd.DataFrame({\n            'video_id': meta_changes.loc[mask, 'video_id'].values.astype(str),  # 确保为字符串\n            'agent_id': meta_changes.loc[mask, 'agent_id'].values,\n            'target_id': meta_changes.loc[mask, 'target_id'].values,\n            'action': pred.columns[ama_changes[mask].values],\n            'start_frame': ama_changes[mask].index.values.astype(int),  # 帧号为整数\n            'stop_frame': ama_changes.index[1:][mask[:-1]].astype(int)\n        })\n        \n        # 7. 修正跨视频/agent/target的结束帧\n        stop_meta = meta_changes.iloc[1:].reset_index(drop=True)\n        stop_video_id = stop_meta.loc[mask[:-1], 'video_id'].values.astype(str)\n        stop_agent_id = stop_meta.loc[mask[:-1], 'agent_id'].values\n        stop_target_id = stop_meta.loc[mask[:-1], 'target_id'].values\n        \n        for i in range(len(submission_part)):\n            if (stop_video_id[i] != submission_part['video_id'].iloc[i] or\n                stop_agent_id[i] != submission_part['agent_id'].iloc[i] or\n                stop_target_id[i] != submission_part['target_id'].iloc[i]):\n                # 修正为当前视频的最大帧+1\n                max_frame = meta.query(\"video_id == @submission_part['video_id'].iloc[i]\")['video_frame'].max()\n                submission_part.iloc[i, submission_part.columns.get_loc('stop_frame')] = int(max_frame) + 1\n    \n    print(f\"\\t\\t✅ predict_multiclass 生成 {len(submission_part)} 行区间结果\")\n    return submission_part","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:10.304822Z","iopub.execute_input":"2025-11-23T14:08:10.305080Z","iopub.status.idle":"2025-11-23T14:08:10.314741Z","shell.execute_reply.started":"2025-11-23T14:08:10.305059Z","shell.execute_reply":"2025-11-23T14:08:10.314049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tune_threshold(oof_action, y_action):\n    def objective(trial):\n        threshold = trial.suggest_float(\"threshold\", 0, 1, step=0.01)\n        return f1_score(y_action, (oof_action >= threshold), zero_division=0)\n\n    study = optuna.create_study(direction=\"maximize\")\n    study.optimize(objective, n_trials=1000, n_jobs=-1)\n    return study.best_params[\"threshold\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:10.611717Z","iopub.execute_input":"2025-11-23T14:08:10.612591Z","iopub.status.idle":"2025-11-23T14:08:10.616950Z","shell.execute_reply.started":"2025-11-23T14:08:10.612565Z","shell.execute_reply":"2025-11-23T14:08:10.616101Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport os\nimport warnings\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import f1_score\nfrom xgboost import XGBClassifier\nfrom catboost import CatBoostClassifier\nimport joblib\nimport torch\n\ndef cross_validate_classifier(X, label, meta, body_parts_tracked_str, section):\n    \"\"\"\n    最终适配版：完全匹配你的 CFG 配置，无任何参数冲突\n    核心：fit 中仅传 eval_set 和 verbose，删除所有构造函数已有的参数\n    \"\"\"\n    # 初始化 OOF 结果（用 meta 原始索引对齐，避免帧错位）\n    oof = pd.DataFrame(index=meta.index)\n    f1_list = []\n    submission_list = []\n    thresholds = {}\n\n    # 遍历每个动作（多标签分类）\n    for action in label.columns:\n        # -----------------------\n        # 1. 数据筛选：过滤空标签和无效数据\n        # -----------------------\n        action_mask = ~label[action].isna().values  # 非空标签掩码\n        y_action = label[action][action_mask].values.astype(int)  # 动作标签（0/1）\n        X_action = X[action_mask]  # 动作特征\n        groups_action = meta.video_id[action_mask]  # 分组（按视频 ID，避免同视频跨折）\n\n        # 至少需要 2 个不同视频才能做 CV（否则训练无意义）\n        if len(np.unique(groups_action)) < 2:\n            print(f\"\\t跳过动作 {action}：视频数量不足 2 个\")\n            continue\n\n        # 非全负样本（全负样本直接 F1=0，无需训练）\n        if not (y_action == 0).all():\n            try:\n                with warnings.catch_warnings():\n                    warnings.filterwarnings(\"ignore\", category=RuntimeWarning)  # 忽略冗余警告\n\n                    # ===========================\n                    # 2. CV 配置：2 折提速（平衡速度与稳定性）\n                    # ===========================\n                    n_splits = min(2, len(np.unique(groups_action)))  # 最多 2 折（覆盖 CFG 的 3 折，优先提速）\n                    cv = GroupKFold(n_splits=n_splits)\n                    oof_action = np.zeros(len(y_action))  # 存储当前动作的 OOF 预测\n                    preds = []  # 存储每折模型（XGB+CatBoost）\n\n                    # ===========================\n                    # 3. 多折训练：XGB 折级轮训 + CatBoost 双 GPU 并行\n                    # ===========================\n                    for fold, (train_idx, valid_idx) in enumerate(cv.split(X_action, y_action, groups_action)):\n                        print(f\"\\t动作 {action} - 折 {fold+1}/{n_splits} 训练中...\")\n\n                        # -----------------------\n                        # XGBoost：折级轮训（双 T4 交替使用）\n                        # -----------------------\n                        current_xgb_params = CFG.xgb_params.copy()\n                        current_xgb_params[\"gpu_id\"] = fold % 2  # 折 0→GPU0，折1→GPU1（覆盖 CFG 的 gpu_id=0）\n                        xgb_model = XGBClassifier(**current_xgb_params)\n                        \n                        # 关键修改：fit 中只传 eval_set 和 verbose，删除所有其他参数！\n                        # 原因：eval_metric、early_stopping_rounds 已在 CFG.xgb_params 中定义（数值 30）\n                        xgb_model.fit(\n                            X_action.iloc[train_idx], y_action[train_idx],\n                            eval_set=[(X_action.iloc[valid_idx], y_action[valid_idx])],\n                            verbose=False  # 仅保留 2 个必要参数，彻底杜绝重复\n                        )\n                        \n                        # 预测（旧版本兼容：用 iteration_range 替代 ntree_limit）\n                        if hasattr(xgb_model, \"best_iteration\") and xgb_model.best_iteration > 0:\n                            xgb_pred = xgb_model.predict_proba(\n                                X_action.iloc[valid_idx],\n                                iteration_range=(0, xgb_model.best_iteration)  # 仅用最优迭代\n                            )[:, 1]\n                        else:\n                            xgb_pred = xgb_model.predict_proba(X_action.iloc[valid_idx])[:, 1]\n\n                        # -----------------------\n                        # CatBoost：双 GPU 并行训练（无需手动切换，直接用 CFG 参数）\n                        # -----------------------\n                        cat_model = CatBoostClassifier(**CFG.cat_params)\n                        cat_model.fit(\n                            X_action.iloc[train_idx], y_action[train_idx],\n                            eval_set=(X_action.iloc[valid_idx], y_action[valid_idx]),\n                            verbose=False\n                        )\n                        \n                        # 预测（用早停后的最优迭代）\n                        if hasattr(cat_model, \"best_iteration\") and cat_model.best_iteration > 0:\n                            cat_pred = cat_model.predict_proba(\n                                X_action.iloc[valid_idx],\n                                ntree_end=cat_model.best_iteration\n                            )[:, 1]\n                        else:\n                            cat_pred = cat_model.predict_proba(X_action.iloc[valid_idx])[:, 1]\n\n                        # -----------------------\n                        # 模型融合：概率平均（简单高效）\n                        # -----------------------\n                        fold_pred = (xgb_pred + cat_pred) / 2\n                        oof_action[valid_idx] = fold_pred  # 回填 OOF 结果\n\n                        # 保存当前折模型（后续提交用）\n                        preds.append((xgb_model, cat_model))\n\n                        # 清理当前折资源（避免显存/内存堆积）\n                        del xgb_model, cat_model, xgb_pred, cat_pred, fold_pred\n                        gc.collect()\n                        if torch.cuda.is_available():\n                            torch.cuda.empty_cache()  # 清理 PyTorch 显存残留\n\n                    # ===========================\n                    # 4. 阈值优化：寻找最优 F1 阈值（避免默认 0.5 偏见）\n                    # ===========================\n                    threshold = tune_threshold(oof_action, y_action)  # 自定义阈值优化函数\n                    thresholds[action] = threshold\n\n                    # 计算 F1 分数（零除保护）\n                    f1 = f1_score(y_action, (oof_action >= threshold), zero_division=0)\n                    f1_list.append((body_parts_tracked_str, action, f1))\n                    print(f\"\\t动作 {action} - F1: {f1:.4f} | 最优阈值: {threshold:.2f}\")\n\n                    # ===========================\n                    # 5. 模型保存：按 section→action 分层存储（后续提交复用）\n                    # ===========================\n                    save_dir = f\"{CFG.model_path}/{CFG.model_name}/{section}/{action}\"\n                    os.makedirs(save_dir, exist_ok=True)\n                    joblib.dump(preds, f\"{save_dir}/xgb_cat_trainer.pkl\")  # 保存所有折模型\n                    joblib.dump(threshold, f\"{save_dir}/best_threshold.pkl\")  # 保存最优阈值\n\n            except Exception as e:\n                # 训练失败时，OOF 填充 0，避免中断整体流程\n                print(f\"\\t动作 {action} 训练失败: {str(e)}\")\n                oof_action = np.zeros(len(y_action))\n        else:\n            # 全负样本：F1=0，阈值=0\n            oof_action = np.zeros(len(y_action))\n            thresholds[action] = 0.0\n            print(f\"\\t动作 {action} - F1: 0.0000 | 全负样本\")\n\n        # -----------------------\n        # 6. OOF 回填：将当前动作的 OOF 结果合并到全局\n        # -----------------------\n        oof_column = pd.Series(0.0, index=meta.index)  # 全局初始化\n        oof_column[action_mask] = oof_action  # 按掩码回填当前动作结果\n        oof[action] = oof_column.values\n\n        # 清理当前动作资源\n        del oof_action, y_action, X_action, groups_action, action_mask, oof_column\n        gc.collect()\n\n    # -----------------------\n    # 7. 生成提交结果（按竞赛格式要求）\n    # -----------------------\n    submission_part = predict_multiclass(oof, meta, thresholds)  # 自定义多标签预测函数\n    submission_list.append(submission_part)\n\n    return submission_list, f1_list, thresholds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:10.770042Z","iopub.execute_input":"2025-11-23T14:08:10.770282Z","iopub.status.idle":"2025-11-23T14:08:14.113816Z","shell.execute_reply.started":"2025-11-23T14:08:10.770265Z","shell.execute_reply":"2025-11-23T14:08:14.112922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def submit(body_parts_tracked_str, switch_tr, section, thresholds):\n    \"\"\"\n    完整预测流程：数据生成→特征转换→模型预测→区间生成\n    输出符合比赛要求的预测批次列表\n    \"\"\"\n    import glob\n    import joblib\n    import pandas as pd\n    import numpy as np\n    import gc\n    \n    # 全局变量依赖（确保提前定义）\n    global drop_body_parts, CFG, test, generate_mouse_data, _fps_from_meta\n    global transform_single, transform_pair, predict_multiclass\n    \n    # 1. 解析并过滤身体部位\n    body_parts_tracked = json.loads(body_parts_tracked_str)\n    if 'drop_body_parts' in globals() and len(body_parts_tracked) > 5:\n        body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n\n    # 2. 筛选当前Section的测试数据\n    test_subset = test[test['body_parts_tracked'] == body_parts_tracked_str]\n    if len(test_subset) == 0:\n        print(f\"\\t⚠️ 当前Section无测试数据，跳过\")\n        return []\n\n    # 3. 初始化数据生成器\n    generator = generate_mouse_data(\n        test_subset,\n        'test',\n        generate_single=(switch_tr == 'single'),\n        generate_pair=(switch_tr == 'pair')\n    )\n\n    # 4. 构建FPS查找字典\n    fps_lookup = (\n        test_subset[['video_id', 'frames_per_second']]\n        .drop_duplicates('video_id')\n        .set_index('video_id')['frames_per_second']\n        .to_dict()\n    )\n\n    submission_list = []\n\n    # 5. 迭代预测\n    for switch_te, data_te, meta_te, actions_te in generator:\n        assert switch_te == switch_tr, f\"数据类型不匹配：{switch_te} != {switch_tr}\"\n        try:\n            # 5.1 获取FPS\n            fps_i = _fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n\n            # 5.2 特征转换\n            if switch_te == 'single':\n                X_te = transform_single(data_te, body_parts_tracked, fps_i).astype(np.float32)\n            else:\n                X_te = transform_pair(data_te, body_parts_tracked, fps_i).astype(np.float32)\n\n            del data_te\n            gc.collect()\n\n            # 5.3 初始化预测概率DataFrame\n            pred_prob = pd.DataFrame(index=meta_te['video_frame'].values)\n            print(f\"\\t\\t✅ 初始化预测结果，帧数量：{len(pred_prob)}\")\n\n            # 5.4 逐个行为预测（XGBoost+CatBoost融合）\n            for action in actions_te:\n                model_files = glob.glob(\n                    f\"{CFG.model_path}/{CFG.model_name}/{section}/{action}/xgb_cat_trainer.pkl\"\n                )\n                if len(model_files) != 1:\n                    print(f\"\\t\\t⚠️ 未找到{action}的模型文件，跳过\")\n                    continue\n\n                models = joblib.load(model_files[0])\n                if not isinstance(models, list) or len(models) == 0:\n                    print(f\"\\t\\t⚠️ {action}模型格式错误，跳过\")\n                    continue\n\n                # 多折融合\n                fold_probs = []\n                for (xgb_model, cat_model) in models:\n                    xgb_prob = xgb_model.predict_proba(X_te)[:, 1]\n                    cat_prob = cat_model.predict_proba(X_te)[:, 1]\n                    fold_probs.append((xgb_prob + cat_prob) / 2)\n\n                pred_prob[action] = np.mean(fold_probs, axis=0)\n                del models, fold_probs\n                gc.collect()\n                print(f\"\\t\\t✅ 完成{action}预测\")\n\n            del X_te\n            gc.collect()\n\n            # 5.5 生成区间结果\n            if pred_prob.shape[1] > 0:\n                submission_part = predict_multiclass(pred_prob, meta_te, thresholds)\n                if len(submission_part) > 0:\n                    submission_list.append(submission_part)\n                    print(f\"\\t\\t✅ 新增预测批次，行数：{len(submission_part)}\")\n\n        except Exception as e:\n            print(f\"\\t⚠️ 处理{switch_te}数据出错：{type(e).__name__}: {str(e)}\")\n            import traceback\n            traceback.print_exc()\n            if 'data_te' in locals():\n                del data_te\n            gc.collect()\n            continue\n\n    print(f\"\\t✅ submit执行完成，生成{len(submission_list)}个有效批次\")\n    return submission_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:14.115234Z","iopub.execute_input":"2025-11-23T14:08:14.115673Z","iopub.status.idle":"2025-11-23T14:08:14.127822Z","shell.execute_reply.started":"2025-11-23T14:08:14.115649Z","shell.execute_reply":"2025-11-23T14:08:14.127055Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport joblib\n\n# 全局定义阈值字典（确保作用域正确）\nthresholds = {\n    \"single\": {},\n    \"pair\": {}\n}\n\nif CFG.mode != \"validate\":\n    # 根路径：models/xgb_cat/\n    thresholds_root = f\"{CFG.model_path}/{CFG.model_name}\"\n    default_threshold = 0.5  # 兜底默认值\n\n    # 关键：行为类型映射表（根据 MABe 挑战的行为定义分类）\n    # 手动将所有行为划分为 single（单鼠行为）和 pair（双鼠交互行为）\n    action_type_map = {\n        # 单鼠行为（single）\n        \"rear\": \"single\",           # 站立\n        \"selfgroom\": \"single\",      # 自我梳理\n        \"rest\": \"single\",           # 休息\n        \"dig\": \"single\",            # 挖掘\n        \"climb\": \"single\",          # 攀爬\n        \"freeze\": \"single\",         # 冻结\n        \"biteobject\": \"single\",     # 咬物体\n        \"exploreobject\": \"single\",  # 探索物体\n        \"run\": \"single\",            # 奔跑\n\n        # 双鼠交互行为（pair）\n        \"approach\": \"pair\",         # 接近\n        \"attack\": \"pair\",           # 攻击\n        \"avoid\": \"pair\",            # 躲避\n        \"chase\": \"pair\",            # 追逐\n        \"chaseattack\": \"pair\",      # 追逐攻击\n        \"huddle\": \"pair\",           # 依偎\n        \"reciprocalsniff\": \"pair\",  # 互相嗅闻\n        \"sniffgenital\": \"pair\",     # 嗅生殖器\n        \"dominance\": \"pair\",        # 支配\n        \"escape\": \"pair\",           # 逃跑\n        \"follow\": \"pair\",           # 跟随\n        \"sniff\": \"pair\",            # 嗅闻\n        \"defend\": \"pair\",           # 防御\n        \"mount\": \"pair\",            # 爬跨\n        \"sniffface\": \"pair\",         # 嗅面部\n        \"sniffbody\": \"pair\",        # 嗅身体\n        \"attemptmount\": \"pair\",     # 尝试爬跨\n        \"shepherd\": \"pair\",         # 驱赶\n        \"allogroom\": \"pair\",        # 互相梳理\n        \"disengage\": \"pair\",        # 脱离\n        \"dominancegroom\": \"pair\",   # 支配性梳理\n        \"dominancemount\": \"pair\",   # 支配性爬跨\n        \"ejaculate\": \"pair\",        # 射精\n        \"genitalgroom\": \"pair\",     # 生殖器梳理\n        \"intromit\": \"pair\"          # 插入\n        # 如有遗漏行为，按相同格式补充\n    }\n\n    # 1. 遍历所有 Section 文件夹（1-9）\n    for section_name in os.listdir(thresholds_root):\n        section_path = os.path.join(thresholds_root, section_name)\n        # 只处理数字命名的 Section 目录（如 1、2、3...）\n        if not (os.path.isdir(section_path) and section_name.isdigit()):\n            continue\n        print(f\"\\n===== 处理 Section {section_name} =====\")\n\n        # 2. 遍历当前 Section 下的所有行为文件夹（如 attack、rear 等）\n        for action_name in os.listdir(section_path):\n            action_path = os.path.join(section_path, action_name)\n            # 跳过 submit 文件夹和非目录文件（只处理行为文件夹）\n            if not os.path.isdir(action_path) or action_name == \"submit\":\n                continue\n\n            # 3. 检查当前行为文件夹下是否有 best_threshold.pkl\n            threshold_file = os.path.join(action_path, \"best_threshold.pkl\")\n            if os.path.exists(threshold_file):\n                print(f\"找到行为 {action_name} 的阈值文件：{threshold_file}\")\n                try:\n                    # 读取阈值（假设 pkl 直接存储数值，如 0.62）\n                    threshold_val = joblib.load(threshold_file)\n                    # 确保读取到的是数值（防止文件格式错误）\n                    if not isinstance(threshold_val, (int, float)):\n                        raise ValueError(\"阈值文件内容不是数值\")\n\n                    # 4. 确定行为类型（single/pair）\n                    if action_name in action_type_map:\n                        behavior_type = action_type_map[action_name]\n                    else:\n                        # 未知行为默认归为 pair（可根据实际情况调整）\n                        behavior_type = \"pair\"\n                        print(f\"⚠️ 未知行为 {action_name}，默认归为 pair 类型\")\n\n                    # 5. 存入阈值字典（若同一行为在多个 Section 出现，保留最后一个值）\n                    thresholds[behavior_type][action_name] = threshold_val\n                    print(f\"✅ 加载成功：{behavior_type}/{action_name} = {threshold_val}\")\n\n                except Exception as e:\n                    print(f\"❌ 读取 {threshold_file} 失败：{str(e)}，使用默认值 {default_threshold}\")\n                    # 读取失败时用默认值兜底\n                    behavior_type = action_type_map.get(action_name, \"pair\")\n                    thresholds[behavior_type][action_name] = default_threshold\n\n    # 6. 校验并补充未加载的行为（使用默认值）\n    print(\"\\n\" + \"=\"*60)\n    print(\"最终阈值汇总：\")\n    print(f\"single 行为（{len(thresholds['single'])} 个）：{thresholds['single'].keys()}\")\n    print(f\"pair 行为（{len(thresholds['pair'])} 个）：{thresholds['pair'].keys()}\")\n\n    # 为所有已知行为设置默认值（确保没有遗漏）\n    for action, btype in action_type_map.items():\n        if action not in thresholds[btype]:\n            thresholds[btype][action] = default_threshold\n            print(f\"⚠️ 行为 {btype}/{action} 未找到阈值文件，使用默认值 {default_threshold}\")\n\nelse:\n    print(\"当前为 validate 模式，使用空阈值字典\")\n\n# 确认变量可用\nprint(f\"\\n阈值字典状态：single={len(thresholds['single'])} 个，pair={len(thresholds['pair'])} 个\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:14.128651Z","iopub.execute_input":"2025-11-23T14:08:14.128925Z","iopub.status.idle":"2025-11-23T14:08:15.326794Z","shell.execute_reply.started":"2025-11-23T14:08:14.128909Z","shell.execute_reply":"2025-11-23T14:08:15.326094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------- 主循环：完整处理流程 + 比赛格式提交 ----------------------\nimport pandas as pd\nimport numpy as np\nimport gc\nimport torch\n\n# 全局变量（确保提前定义）\n# CFG, drop_body_parts, thresholds, action_type_map, body_parts_tracked_list\n# test, generate_mouse_data, _fps_from_meta, transform_single, transform_pair\n# predict_multiclass, submit, robustify\n\n# 读取测试集元数据\ntest_meta = pd.read_csv(\"/kaggle/input/MABe-mouse-behavior-detection/test.csv\")\nprint(f\"成功读取测试集元数据，共{len(test_meta)}条记录\")\n\n# 初始化结果列表\nsubmission_list = []\n\n# 处理所有Section\ntotal_sections = len(body_parts_tracked_list) - 1\nprint(f\"开始处理{total_sections}个Section的视频数据...\\n\")\n\nfor section in range(1, len(body_parts_tracked_list)):\n    body_parts_tracked_str = body_parts_tracked_list[section]\n    section_str = f\"{section}\"\n    \n    try:\n        # 1. 解析身体部位\n        body_parts_tracked = json.loads(body_parts_tracked_str)\n        print(f\"===== {section}/{total_sections} 处理中 =====\")\n        print(f\"追踪的身体部位：{body_parts_tracked}\")\n        \n        # 2. 过滤身体部位\n        if 'drop_body_parts' in globals() and len(body_parts_tracked) > 5:\n            body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n            print(f\"过滤后保留：{body_parts_tracked}\")\n        \n        # 3. 筛选测试数据\n        data_subset = test_meta[test_meta['body_parts_tracked'].astype(str) == body_parts_tracked_str].copy()\n        print(f\"\\t✅ 筛选测试数据：{len(data_subset)}条\")\n        \n        if len(data_subset) == 0:\n            print(f\"\\t⚠️ 无有效数据，跳过\\n\")\n            continue\n        \n        # 4. 调用submit函数预测（单鼠+双鼠）\n        # 单鼠预测\n        single_sub = submit(\n            body_parts_tracked_str=body_parts_tracked_str,\n            switch_tr='single',\n            section=section,\n            thresholds=thresholds[\"single\"].get(section_str, {})\n        )\n        if single_sub:\n            submission_list.extend(single_sub)\n            print(f\"\\t✅ 单鼠预测新增{len(single_sub)}个批次\")\n        \n        # 双鼠预测\n        pair_sub = submit(\n            body_parts_tracked_str=body_parts_tracked_str,\n            switch_tr='pair',\n            section=section,\n            thresholds=thresholds[\"pair\"].get(section_str, {})\n        )\n        if pair_sub:\n            submission_list.extend(pair_sub)\n            print(f\"\\t✅ 双鼠预测新增{len(pair_sub)}个批次\")\n        \n        # 清理内存\n        gc.collect()\n        if torch.cuda.is_available():\n            torch.cuda.empty_cache()\n        \n        print(f\"===== {section}/{total_sections} 处理完成 =====\\n\")\n    \n    except Exception as e:\n        print(f\"\\t❌ 处理Section{section}出错：{type(e).__name__}: {str(e)}\")\n        import traceback\n        traceback.print_exc()\n        print(f\"===== {section}/{total_sections} 处理失败 =====\\n\")\n        continue\n\n# ---------------------- 比赛要求的提交逻辑 ----------------------\nprint(\"=\"*50)\nprint(\"所有Section处理完成！开始生成提交文件...\")\n\nif CFG.mode == 'submit':\n    # 合并所有预测结果\n    if len(submission_list) > 0:\n        submission = pd.concat(submission_list, ignore_index=True)\n        print(f\"合并后预测结果行数：{len(submission)}\")\n    else:\n        # 无预测结果时，生成默认空结果（比赛要求的格式）\n        submission = pd.DataFrame(\n            dict(\n                video_id='438887472',  # 字符串格式\n                agent_id='mouse1',\n                target_id='self',\n                action='rear',\n                start_frame=278,  # 整数格式\n                stop_frame=500\n            ),\n            index=[0]  # 索引为整数\n        )\n        print(f\"无有效预测结果，生成默认格式占位符\")\n    \n    # 比赛要求的robustify后处理\n    submission_robust = robustify(submission, test, 'test')\n    \n    # 设置索引名称为row_id（比赛要求）\n    submission_robust.index.name = 'row_id'\n    \n    # 保存提交文件\n    submission_robust.to_csv('submission.csv')\n    print(f\"\\n✅ 提交文件生成成功！\")\n    print(f\"提交文件形状：{submission_robust.shape}\")\n    print(f\"提交文件列名：{list(submission_robust.columns)}\")\n    print(f\"提交文件前5行：\\n{submission_robust.head()}\")\n\nprint(\"=\"*50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:08:15.328321Z","iopub.execute_input":"2025-11-23T14:08:15.328578Z","iopub.status.idle":"2025-11-23T14:09:40.465618Z","shell.execute_reply.started":"2025-11-23T14:08:15.328560Z","shell.execute_reply":"2025-11-23T14:09:40.464783Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if CFG.mode == 'validate':  \n    submission = pd.concat(submission_list)\n    submission_robust = robustify(submission, train, 'train')\n    print(f\"Competition metric: {score(solution, submission_robust, ''):.4f}\")\n\n    f1_df = pd.DataFrame(f1_list, columns=['body_parts_tracked_str', 'action'    , 'binary F1 score'])\n    print(f\"Mean F1:            {f1_df['binary F1 score'].mean():.4f}\")\n  \n    joblib.dump(thresholds, f\"{CFG.model_name}/thresholds.pkl\")\n    joblib.dump(f1_df, f\"{CFG.model_name}/scores.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:09:40.466472Z","iopub.execute_input":"2025-11-23T14:09:40.466796Z","iopub.status.idle":"2025-11-23T14:09:40.471475Z","shell.execute_reply.started":"2025-11-23T14:09:40.466768Z","shell.execute_reply":"2025-11-23T14:09:40.470817Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"if CFG.mode == 'submit':\n    if len(submission_list) > 0:\n        submission = pd.concat(submission_list)\n    else:\n        submission = pd.DataFrame(\n            dict(\n                video_id=438887472,\n                agent_id='mouse1',\n                target_id='self',\n                action='rear',\n                start_frame='278',\n                stop_frame='500'\n            ), index=[44])\n        \n    submission_robust = robustify(submission, test, 'test')\n    submission_robust.index.name = 'row_id'\n    submission_robust.to_csv('submission.csv')\n    submission.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:09:40.472204Z","iopub.execute_input":"2025-11-23T14:09:40.472480Z","iopub.status.idle":"2025-11-23T14:09:40.614581Z","shell.execute_reply.started":"2025-11-23T14:09:40.472457Z","shell.execute_reply":"2025-11-23T14:09:40.613828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T14:09:40.615376Z","iopub.execute_input":"2025-11-23T14:09:40.615666Z","iopub.status.idle":"2025-11-23T14:09:40.632804Z","shell.execute_reply.started":"2025-11-23T14:09:40.615643Z","shell.execute_reply":"2025-11-23T14:09:40.632110Z"}},"outputs":[],"execution_count":null}]}