{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13822995,"sourceType":"datasetVersion","datasetId":8802877}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":1494.813501,"end_time":"2025-10-25T19:51:35.870094","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-10-25T19:26:41.056593","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"00962fe9010e42b7a5b123af50d2a026":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"0b098fa23a3b4031bf88b32b20044978":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"108c3e5db1154f47a5aca68386ed4f24":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"38c36db3be3843b19da108d4cd1f9421":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3e9806911d1d489d963f76955d8468c4","IPY_MODEL_e6ad60bc760e4b51885b15335c048a65","IPY_MODEL_6f4723180bf14101b60deeac9fd59e50"],"layout":"IPY_MODEL_00962fe9010e42b7a5b123af50d2a026","tabbable":null,"tooltip":null}},"3e9806911d1d489d963f76955d8468c4":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_aeb2544f9ad9413b8b71b5f5643b1019","placeholder":"​","style":"IPY_MODEL_ec012e9ac53d477daee0988a35117f0b","tabbable":null,"tooltip":null,"value":"100%"}},"3edd91e1eafb4f6bbf55275ea60b6c97":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f4723180bf14101b60deeac9fd59e50":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_e8f95c8096294ce6b6fb2b65c08af536","placeholder":"​","style":"IPY_MODEL_0b098fa23a3b4031bf88b32b20044978","tabbable":null,"tooltip":null,"value":" 863/863 [00:08&lt;00:00, 96.46it/s]"}},"aeb2544f9ad9413b8b71b5f5643b1019":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e6ad60bc760e4b51885b15335c048a65":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_3edd91e1eafb4f6bbf55275ea60b6c97","max":863,"min":0,"orientation":"horizontal","style":"IPY_MODEL_108c3e5db1154f47a5aca68386ed4f24","tabbable":null,"tooltip":null,"value":863}},"e8f95c8096294ce6b6fb2b65c08af536":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"ec012e9ac53d477daee0988a35117f0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install catboost","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T01:53:08.868853Z","iopub.execute_input":"2025-12-30T01:53:08.869538Z","iopub.status.idle":"2025-12-30T01:53:12.745133Z","shell.execute_reply.started":"2025-12-30T01:53:08.869511Z","shell.execute_reply":"2025-12-30T01:53:12.744467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\nimport gc\nimport itertools\nimport warnings\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.base import clone\nfrom lightgbm import LGBMClassifier\n\n# ==========================================\n# 1. 引入 CatBoost\n# ==========================================\ntry:\n    from catboost import CatBoostClassifier\n    print(\"CatBoost imported successfully.\")\nexcept ImportError:\n    raise ImportError(\"请先安装 CatBoost: !pip install catboost\")\n\nsns.set_style(\"whitegrid\")\nplt.rcParams['font.sans-serif'] = ['DejaVu Sans']\nwarnings.filterwarnings('ignore')\n\n# ==========================================\n# 2. 配置 (Configuration)\n# ==========================================\nclass CFG:\n    train_path = \"/kaggle/input/MABe-mouse-behavior-detection/train.csv\"\n    tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n    annotation_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n    seed = 42\n    \n    # 内存安全设置：\n    # 建议先用 20-50 个视频测试能否跑通，确认无误后可设为 100 或 None(全量)\n    MAX_TRAIN_VIDEOS = 50 \n    MAX_VAL_VIDEOS = 20\n    \n    FIXED_BODY_PARTS = [\n        'nose', 'ear_left', 'ear_right', 'neck', \n        'body_center', 'lateral_left', 'lateral_right', \n        'tail_base', 'tail_midpoint', 'tail_tip',\n        'hip_left', 'hip_right', 'spine_1', 'spine_2'\n    ]\n\n# ==========================================\n# 3. 特征工程 (保持不变)\n# ==========================================\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef add_physics_features(X, cx, cy, fps):\n    vel_x = cx.diff().fillna(0)\n    vel_y = cy.diff().fillna(0)\n    speed = np.sqrt(vel_x**2 + vel_y**2) * float(fps)\n    for w in [5, 15, 30]:\n        ws = _scale(w, fps)\n        X[f'speed_mean_{w}'] = speed.rolling(ws, min_periods=1, center=True).mean().fillna(0)\n        X[f'speed_std_{w}'] = speed.rolling(ws, min_periods=1, center=True).std().fillna(0)\n    return X\n\ndef transform_single(single_mouse, fps):\n    combos = list(itertools.combinations(CFG.FIXED_BODY_PARTS, 2))\n    dist_cols = [f\"{p1}+{p2}\" for p1, p2 in combos]\n    X = pd.DataFrame(0.0, index=single_mouse.index, columns=dist_cols, dtype='float32')\n    available_parts = set(single_mouse.columns.levels[0])\n    \n    for p1, p2 in combos:\n        if p1 in available_parts and p2 in available_parts:\n            dist = np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1)\n            X[f\"{p1}+{p2}\"] = dist.astype('float32')\n            \n    if 'body_center' in available_parts:\n        X = add_physics_features(X, single_mouse['body_center']['x'], single_mouse['body_center']['y'], fps)\n    elif 'nose' in available_parts:\n         X = add_physics_features(X, single_mouse['nose']['x'], single_mouse['nose']['y'], fps)\n    \n    return X.fillna(0.0)\n\n# ==========================================\n# 4. 数据生成器\n# ==========================================\ndef generate_mouse_data_train(dataset, tracking_dir, annot_dir, max_videos=None):\n    count = 0\n    for _, row in dataset.iterrows():\n        if max_videos and count >= max_videos: break\n        \n        lab_id, video_id = row.lab_id, row.video_id\n        try:\n            path = f\"{tracking_dir}/{lab_id}/{video_id}.parquet\"\n            vid = pd.read_parquet(path)\n            pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n            pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n            pvid /= row.pix_per_cm_approx\n            \n            a_path = f\"{annot_dir}/{lab_id}/{video_id}.parquet\"\n            annot = pd.read_parquet(a_path)\n            fps = row.frames_per_second\n            mouse_ids = pvid.columns.levels[0].unique()\n\n            for mid in mouse_ids:\n                if mid not in pvid.columns: continue\n                X = transform_single(pvid[mid], fps)\n                my_annot = annot[(annot.agent_id == mid)]\n                \n                if len(my_annot) > 0:\n                    y = pd.DataFrame(0, index=X.index, columns=my_annot.action.unique(), dtype='int8')\n                    for _, ar in my_annot.iterrows():\n                        start, stop = max(0, ar.start_frame), min(len(X)-1, ar.stop_frame)\n                        if start < stop:\n                            y.loc[start:stop, ar.action] = 1\n                    yield X, y, video_id\n            \n            count += 1\n            if count % 10 == 0: gc.collect()\n        except Exception:\n            continue\n\n# ==========================================\n# 5. 主实验逻辑 (CatBoost vs LightGBM)\n# ==========================================\ndef run_catboost_vs_lgbm():\n    print(\"Step 1: Setup...\")\n    train_df = pd.read_csv(CFG.train_path)\n    train_df = train_df[~train_df.lab_id.str.startswith('MABe22')]\n    \n    all_videos = train_df.video_id.unique()\n    np.random.seed(CFG.seed)\n    val_video_ids = np.random.choice(all_videos, size=int(len(all_videos)*0.2), replace=False)\n    \n    train_subset = train_df[~train_df.video_id.isin(val_video_ids)]\n    val_subset = train_df[train_df.video_id.isin(val_video_ids)]\n    \n    print(f\"Limiting usage to: Train={CFG.MAX_TRAIN_VIDEOS}, Val={CFG.MAX_VAL_VIDEOS}\")\n    \n    # --- 1. Load Training Data ---\n    print(\"Step 2: Loading Training Data...\")\n    X_train_list, y_train_list = [], []\n    gen = generate_mouse_data_train(train_subset, CFG.tracking_path, CFG.annotation_path, max_videos=CFG.MAX_TRAIN_VIDEOS)\n    \n    for X, y, vid in gen:\n        X_train_list.append(X)\n        y_train_list.append(y)\n    \n    X_train = pd.concat(X_train_list).reset_index(drop=True)\n    y_train = pd.concat(y_train_list).fillna(0).reset_index(drop=True)\n    del X_train_list, y_train_list\n    gc.collect()\n    print(f\"Training Data Loaded. Shape: {X_train.shape}\")\n    \n    # --- 2. Cache Validation Data ---\n    print(\"Step 2.5: Caching Validation Data...\")\n    X_val_cache = []\n    gen_val = generate_mouse_data_train(val_subset, CFG.tracking_path, CFG.annotation_path, max_videos=CFG.MAX_VAL_VIDEOS)\n    for X_val, y_val, vid in gen_val:\n        X_val_cache.append((X_val, y_val, vid))\n    print(f\"Validation Cache Ready: {len(X_val_cache)} items.\")\n\n    # ==========================================\n    # 【核心修改】定义模型对比池\n    # ==========================================\n    BASE_MODELS = {\n        'LightGBM': LGBMClassifier(\n            n_estimators=100, \n            max_depth=5, \n            num_leaves=31, \n            random_state=42, \n            verbose=-1\n        ),\n        \n        'CatBoost': CatBoostClassifier(\n            iterations=100,       # 迭代次数 (类似 n_estimators)\n            depth=6,              # 树深\n            learning_rate=0.1,\n            loss_function='Logloss',\n            verbose=0,            # 静默训练\n            random_seed=42,\n            allow_writing_files=False # 防止在目录下生成大量日志文件\n        )\n    }\n    \n    # --- 3. Train & Evaluate Loop ---\n    print(\"Step 3: Train-Eval Loop...\")\n    results = []\n    MIN_SAMPLES = 50\n    actions = y_train.columns.tolist()\n    \n    for i, action in enumerate(actions):\n        if y_train[action].sum() < MIN_SAMPLES: continue\n        print(f\"  [{i+1}/{len(actions)}] Action: {action} (n={y_train[action].sum()})\")\n        y_target = y_train[action]\n        \n        for model_name, base_model in BASE_MODELS.items():\n            # A. 训练\n            # CatBoost 需要特别处理一下 fit，因为它有时不完全兼容 clone\n            if model_name == 'CatBoost':\n                clf = CatBoostClassifier(\n                    iterations=100, depth=6, learning_rate=0.1, \n                    verbose=0, random_seed=42, allow_writing_files=False\n                )\n            else:\n                clf = clone(base_model)\n                \n            clf.fit(X_train, y_target)\n            \n            # B. 验证\n            for X_v, y_v_full, vid in X_val_cache:\n                if action not in y_v_full.columns: continue\n                gt_frames = set(y_v_full[y_v_full[action] == 1].index)\n                if len(gt_frames) == 0: continue\n                \n                prob = clf.predict_proba(X_v)[:, 1]\n                pred_frames = set(np.where(prob > 0.3)[0])\n                \n                results.append({\n                    'Action': action, 'Model': model_name,\n                    'TP': len(pred_frames & gt_frames),\n                    'FP': len(pred_frames - gt_frames),\n                    'FN': len(gt_frames - pred_frames)\n                })\n            \n            # C. 释放内存\n            del clf\n            gc.collect()\n            \n    return pd.DataFrame(results)\n\n# ==========================================\n# 6. 绘图\n# ==========================================\ndef plot_full_analysis(results_df):\n    if results_df.empty:\n        print(\"No results to plot.\")\n        return\n        \n    grouped = results_df.groupby(['Action', 'Model']).sum()[['TP', 'FP', 'FN']]\n    grouped['Precision'] = grouped['TP'] / (grouped['TP'] + grouped['FP'] + 1e-6)\n    grouped['Recall'] = grouped['TP'] / (grouped['TP'] + grouped['FN'] + 1e-6)\n    grouped['F1'] = 2 * grouped['Precision'] * grouped['Recall'] / (grouped['Precision'] + grouped['Recall'] + 1e-6)\n    \n    df_plot = grouped.reset_index()\n    \n    # 1. Heatmap\n    plt.figure(figsize=(10, max(4, len(df_plot.Action.unique()) * 0.8)))\n    heatmap_data = df_plot.pivot(index='Action', columns='Model', values='F1')\n    \n    sns.heatmap(heatmap_data, annot=True, cmap='RdYlGn', fmt='.3f', linewidths=.5)\n    plt.title('LightGBM vs CatBoost Performance (F1 Score)')\n    plt.tight_layout()\n    plt.show()\n\n    # 2. Overall Comparison\n    avg_scores = df_plot.groupby('Model')['F1'].mean()\n    print(\"\\n=== Average F1 Score ===\")\n    print(avg_scores)\n\nif __name__ == \"__main__\":\n    try:\n        res = run_catboost_vs_lgbm()\n        plot_full_analysis(res)\n    except Exception as e:\n        print(f\"Error: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-30T01:53:12.746526Z","iopub.execute_input":"2025-12-30T01:53:12.746928Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}