{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14139362,"sourceType":"datasetVersion","datasetId":9010501}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Import và cấu hình","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport glob\nimport json\nimport itertools\nimport warnings\n\nimport numpy as np\nimport pandas as pd\n\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import f1_score\nfrom lightgbm import LGBMClassifier\nimport joblib\n\nwarnings.filterwarnings(\"ignore\")\n\nclass MABe:\n    # Đường dẫn data competition\n    train_path = \"/kaggle/input/MABe-mouse-behavior-detection/train.csv\"\n    test_path = \"/kaggle/input/MABe-mouse-behavior-detection/test.csv\"\n    train_annotation_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n    train_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n    test_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\"\n\n    model_root = \"/kaggle/input/models\"\n    model_subdir = \"lightgbm\"        # folder con trong model_root\n\n    # \"validate\"  -> train + tìm threshold + lưu model/threshold\n    # \"submit\"    -> chỉ load model có sẵn và tạo submission\n    mode = \"submit\"\n\n    # Tham số LightGBM (CPU)\n    lgb_params = dict(\n        n_estimators=500,\n        learning_rate=0.05,\n        max_depth=6,\n        num_leaves=31,\n        min_child_samples=20,\n        subsample=0.8,\n        colsample_bytree=0.8,\n        reg_alpha=0.1,\n        reg_lambda=0.1,\n        random_state=42,\n        n_jobs=-1,\n        verbosity=-1\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:02.989894Z","iopub.execute_input":"2025-12-13T09:40:02.990616Z","iopub.status.idle":"2025-12-13T09:40:13.715971Z","shell.execute_reply.started":"2025-12-13T09:40:02.990583Z","shell.execute_reply":"2025-12-13T09:40:13.715081Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Đọc dữ liệu","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(MABe.train_path)\ntest = pd.read_csv(MABe.test_path)\n\n# List các kiểu body parts\nbody_parts_tracked_list = list(np.unique(train.body_parts_tracked))\n\n#drop các phần body parts khi quá nhiều\ndrop_body_parts = [\n    \"headpiece_bottombackleft\", \"headpiece_bottombackright\",\n    \"headpiece_bottomfrontleft\", \"headpiece_bottomfrontright\",\n    \"headpiece_topbackleft\", \"headpiece_topbackright\",\n    \"headpiece_topfrontleft\", \"headpiece_topfrontright\",\n    \"spine_1\", \"spine_2\",\n    \"tail_middle_1\", \"tail_middle_2\", \"tail_midpoint\"\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.717555Z","iopub.execute_input":"2025-12-13T09:40:13.718510Z","iopub.status.idle":"2025-12-13T09:40:13.865839Z","shell.execute_reply.started":"2025-12-13T09:40:13.718484Z","shell.execute_reply":"2025-12-13T09:40:13.865007Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"code","source":"#Quy đổi window/lag tính theo số frame ở 30fps sang số frame tương ứng ở fps thực tế của video\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\n#Tương tự với scale nhưng giữ dấu (+/0) để dùng cho các offset (quá khứ/ tương lai)\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n    if n_frames_at_30fps == 0:\n        return 0\n    s = 1 if n_frames_at_30fps > 0 else -1\n    mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n    return s * mag\n\n#Lấy fps thật của video\ndef _fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n    if \"frames_per_second\" in meta_df.columns and pd.notnull(meta_df[\"frames_per_second\"]).any():\n        return float(meta_df[\"frames_per_second\"].iloc[0])\n    vid = meta_df[\"video_id\"].iloc[0]\n    return float(fallback_lookup.get(vid, default_fps))\n\n#Sinh dữ liệu theo single và pair mouse\ndef generate_mouse_data(\n    dataset,\n    traintest,                     # 'train' hoặc 'test'\n    traintest_directory=None,\n    generate_single=True,\n    generate_pair=True\n):\n    \"\"\"\n    Sinh dữ liệu:\n      - 'single': từng con chuột (agent = target = self)\n      - 'pair'  : cặp chuột (agent != target)\n    Trả về generator: (switch, data, meta, label / actions)\n    \"\"\"\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n    for _, row in dataset.iterrows():\n        lab_id = row.lab_id\n        if lab_id.startswith(\"MABe22\"):\n            continue\n\n        if type(row.behaviors_labeled) != str:\n            continue\n\n        video_id = row.video_id\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n\n        # Nếu bodypart quá nhiều thì bỏ bớt\n        if len(np.unique(vid.bodypart)) > 5:\n            vid = vid.query(\"~ bodypart.isin(@drop_body_parts)\")\n\n        # Pivot (video_frame, (mouse_id, bodypart, (x,y)))\n        pvid = vid.pivot(\n            columns=[\"mouse_id\", \"bodypart\"],\n            index=\"video_frame\",\n            values=[\"x\", \"y\"]\n        )\n        del vid\n        gc.collect()\n\n        # Đưa về dạng multiindex: bodypart -> coord -> mouse_id\n        pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n\n        # Đổi đơn vị sang cm\n        pvid /= row.pix_per_cm_approx\n\n        # Parse behaviors\n        vid_behaviors = json.loads(row.behaviors_labeled)\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(\",\") for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=[\"agent\", \"target\", \"action\"])\n\n        # Annotation (train mới có)\n        if traintest == \"train\":\n            try:\n                annot = pd.read_parquet(\n                    path.replace(\"train_tracking\", \"train_annotation\")\n                )\n            except FileNotFoundError:\n                continue\n\n        # ---------------- SINGLE (agent = self) ----------------\n        if generate_single:\n            vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n            for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n                try:\n                    mouse_id = int(mouse_id_str[-1])\n                    vid_agent_actions = np.unique(\n                        vid_behaviors_subset.query(\"agent == @mouse_id_str\").action\n                    )\n                    single_mouse = pvid.loc[:, mouse_id]\n                    single_mouse_meta = pd.DataFrame({\n                        \"video_id\": video_id,\n                        \"agent_id\": mouse_id_str,\n                        \"target_id\": \"self\",\n                        \"video_frame\": single_mouse.index\n                    })\n\n                    if traintest == \"train\":\n                        single_mouse_label = pd.DataFrame(\n                            0.0,\n                            columns=vid_agent_actions,\n                            index=single_mouse.index\n                        )\n                        annot_subset = annot.query(\n                            \"(agent_id == @mouse_id) & (target_id == @mouse_id)\"\n                        )\n                        for i in range(len(annot_subset)):\n                            r = annot_subset.iloc[i]\n                            single_mouse_label.loc[r[\"start_frame\"]:r[\"stop_frame\"], r.action] = 1.0\n                        yield \"single\", single_mouse, single_mouse_meta, single_mouse_label\n                    else:\n                        # test: chỉ trả về list action\n                        yield \"single\", single_mouse, single_mouse_meta, vid_agent_actions\n                except KeyError:\n                    pass\n\n        # ---------------- PAIR (agent != target) ----------------\n        if generate_pair:\n            vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n            if len(vid_behaviors_subset) > 0:\n                for agent, target in itertools.permutations(\n                    np.unique(pvid.columns.get_level_values(\"mouse_id\")), 2\n                ):\n                    agent_str = f\"mouse{agent}\"\n                    target_str = f\"mouse{target}\"\n                    vid_agent_actions = np.unique(\n                        vid_behaviors_subset\n                        .query(\"(agent == @agent_str) & (target == @target_str)\")\n                        .action\n                    )\n                    mouse_pair = pd.concat(\n                        [pvid[agent], pvid[target]],\n                        axis=1,\n                        keys=[\"A\", \"B\"]\n                    )\n                    mouse_pair_meta = pd.DataFrame({\n                        \"video_id\": video_id,\n                        \"agent_id\": agent_str,\n                        \"target_id\": target_str,\n                        \"video_frame\": mouse_pair.index\n                    })\n\n                    if traintest == \"train\":\n                        mouse_pair_label = pd.DataFrame(\n                            0.0,\n                            columns=vid_agent_actions,\n                            index=mouse_pair.index\n                        )\n                        annot_subset = annot.query(\n                            \"(agent_id == @agent) & (target_id == @target)\"\n                        )\n                        for i in range(len(annot_subset)):\n                            r = annot_subset.iloc[i]\n                            mouse_pair_label.loc[r[\"start_frame\"]:r[\"stop_frame\"], r.action] = 1.0\n                        yield \"pair\", mouse_pair, mouse_pair_meta, mouse_pair_label\n                    else:\n                        yield \"pair\", mouse_pair, mouse_pair_meta, vid_agent_actions","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.866713Z","iopub.execute_input":"2025-12-13T09:40:13.866958Z","iopub.status.idle":"2025-12-13T09:40:13.884934Z","shell.execute_reply.started":"2025-12-13T09:40:13.866939Z","shell.execute_reply":"2025-12-13T09:40:13.884022Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"## Single-mouse features","metadata":{}},{"cell_type":"code","source":"def add_curvature_features(X, cx, cy, fps):\n    vx = cx.diff()\n    vy = cy.diff()\n    ax = vx.diff()\n    ay = vy.diff()\n\n    cross_prod = vx * ay - vy * ax\n    v_mag = np.sqrt(vx**2 + vy**2)\n    curv = np.abs(cross_prod) / (v_mag**3 + 1e-6)\n\n    for w in [25, 50, 75]:\n        ws = _scale(w, fps)\n        X[f\"curv_mean_{w}\"] = curv.rolling(ws, min_periods=max(1, ws // 5)).mean()\n\n    ang = np.arctan2(vy, vx)\n    ang_diff = np.abs(ang.diff())\n    w = 30\n    ws = _scale(w, fps)\n    X[f\"turn_rate_{w}\"] = ang_diff.rolling(ws, min_periods=max(1, ws // 5)).sum()\n    return X\n\n\ndef add_multiscale_features(X, cx, cy, fps):\n    speed = np.sqrt(cx.diff()**2 + cy.diff()**2) * float(fps)\n    scales = [20, 40, 60, 80]\n    for s in scales:\n        ws = _scale(s, fps)\n        if len(speed) >= ws:\n            X[f\"sp_m{s}\"] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n            X[f\"sp_s{s}\"] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n\n    if len(scales) >= 2 and f\"sp_m{scales[0]}\" in X.columns and f\"sp_m{scales[-1]}\" in X.columns:\n        X[\"sp_ratio\"] = X[f\"sp_m{scales[0]}\"] / (X[f\"sp_m{scales[-1]}\"] + 1e-6)\n    return X\n\n\ndef add_state_features(X, cx, cy, fps):\n    speed = np.sqrt(cx.diff()**2 + cy.diff()**2) * float(fps)\n    w_ma = _scale(15, fps)\n    speed_ma = speed.rolling(w_ma, min_periods=max(1, w_ma // 3)).mean()\n    try:\n        bins = [-np.inf, 0.5, 2.0, 5.0, np.inf]\n        states = pd.cut(speed_ma, bins=bins, labels=[0, 1, 2, 3]).astype(float)\n\n        for window in [20, 40, 60, 80]:\n            ws = _scale(window, fps)\n            if len(states) >= ws:\n                for st in [0, 1, 2, 3]:\n                    X[f\"s{st}_{window}\"] = (\n                        (states == st).astype(float)\n                        .rolling(ws, min_periods=max(1, ws // 5)).mean()\n                    )\n                change = (states != states.shift(1)).astype(float)\n                X[f\"trans_{window}\"] = change.rolling(ws, min_periods=max(1, ws // 5)).sum()\n    except Exception:\n        pass\n    return X\n\n\ndef add_longrange_features(X, cx, cy, fps):\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(cx) >= ws:\n            X[f\"x_ml{window}\"] = cx.rolling(ws, min_periods=max(5, ws // 6)).mean()\n            X[f\"y_ml{window}\"] = cy.rolling(ws, min_periods=max(5, ws // 6)).mean()\n\n    speed = np.sqrt(cx.diff()**2 + cy.diff()**2) * float(fps)\n    for span in [30, 60, 120]:\n        s = _scale(span, fps)\n        X[f\"x_e{span}\"] = cx.ewm(span=s, min_periods=1).mean()\n        X[f\"y_e{span}\"] = cy.ewm(span=s, min_periods=1).mean()\n    for window in [30, 60, 120]:\n        ws = _scale(window, fps)\n        if len(speed) >= ws:\n            X[f\"sp_pct{window}\"] = speed.rolling(ws, min_periods=max(5, ws // 6)).rank(pct=True)\n    return X\n\ndef transform_single(single_mouse, body_parts_tracked, fps):\n    \"\"\"\n    Feature cho 1 con chuột (single).\n    \"\"\"\n    available = single_mouse.columns.get_level_values(0)\n\n    # Khoảng cách bình phương giữa các bodypart\n    X = pd.DataFrame({\n        f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n        if p1 in available and p2 in available\n    })\n    X = X.reindex(\n        columns=[f\"{p1}+{p2}\" for p1, p2 in itertools.combinations(body_parts_tracked, 2)],\n        copy=False\n    )\n\n    # Một số feature tốc độ / hình dạng\n    if all(p in single_mouse.columns for p in [\"ear_left\", \"ear_right\", \"tail_base\"]):\n        lag = _scale(10, fps)\n        shifted = single_mouse[[\"ear_left\", \"ear_right\", \"tail_base\"]].shift(lag)\n        speeds = pd.DataFrame({\n            \"sp_lf\": np.square(single_mouse[\"ear_left\"] - shifted[\"ear_left\"]).sum(axis=1, skipna=False),\n            \"sp_rt\": np.square(single_mouse[\"ear_right\"] - shifted[\"ear_right\"]).sum(axis=1, skipna=False),\n            \"sp_lf2\": np.square(single_mouse[\"ear_left\"] - shifted[\"tail_base\"]).sum(axis=1, skipna=False),\n            \"sp_rt2\": np.square(single_mouse[\"ear_right\"] - shifted[\"tail_base\"]).sum(axis=1, skipna=False),\n        })\n        X = pd.concat([X, speeds], axis=1)\n\n    if \"nose+tail_base\" in X.columns and \"ear_left+ear_right\" in X.columns:\n        X[\"elong\"] = X[\"nose+tail_base\"] / (X[\"ear_left+ear_right\"] + 1e-6)\n\n    if all(p in available for p in [\"nose\", \"body_center\", \"tail_base\"]):\n        v1 = single_mouse[\"nose\"] - single_mouse[\"body_center\"] #tính vector\n        v2 = single_mouse[\"tail_base\"] - single_mouse[\"body_center\"]\n        X[\"body_ang\"] = (v1[\"x\"] * v2[\"x\"] + v1[\"y\"] * v2[\"y\"]) / (\n            np.sqrt(v1[\"x\"]**2 + v1[\"y\"]**2) * np.sqrt(v2[\"x\"]**2 + v2[\"y\"]**2) + 1e-6\n        ) #tính cos (v1,v2)\n\n    if \"body_center\" in available:\n        cx = single_mouse[\"body_center\"][\"x\"]\n        cy = single_mouse[\"body_center\"][\"y\"]\n\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"cx_m{w}\"] = cx.rolling(ws, **roll).mean()\n            X[f\"cy_m{w}\"] = cy.rolling(ws, **roll).mean()\n            X[f\"cx_s{w}\"] = cx.rolling(ws, **roll).std()\n            X[f\"cy_s{w}\"] = cy.rolling(ws, **roll).std()\n            X[f\"x_rng{w}\"] = cx.rolling(ws, **roll).max() - cx.rolling(ws, **roll).min()\n            X[f\"y_rng{w}\"] = cy.rolling(ws, **roll).max() - cy.rolling(ws, **roll).min()\n            X[f\"disp{w}\"] = np.sqrt(\n                cx.diff().rolling(ws, min_periods=1).sum()**2 +\n                cy.diff().rolling(ws, min_periods=1).sum()**2\n            )\n            X[f\"act{w}\"] = np.sqrt(\n                cx.diff().rolling(ws, min_periods=1).var() +\n                cy.diff().rolling(ws, min_periods=1).var()\n            )\n\n        X = add_curvature_features(X, cx, cy, fps)\n        X = add_multiscale_features(X, cx, cy, fps)\n        X = add_state_features(X, cx, cy, fps)\n        X = add_longrange_features(X, cx, cy, fps)\n\n    if all(p in available for p in [\"nose\", \"tail_base\"]):\n        nt_dist = np.sqrt(\n            (single_mouse[\"nose\"][\"x\"] - single_mouse[\"tail_base\"][\"x\"])**2 +\n            (single_mouse[\"nose\"][\"y\"] - single_mouse[\"tail_base\"][\"y\"])**2\n        )\n        for lag in [10, 20, 40]:\n            l = _scale(lag, fps)\n            X[f\"nt_lg{lag}\"] = nt_dist.shift(l)\n            X[f\"nt_df{lag}\"] = nt_dist - nt_dist.shift(l)\n\n    if all(p in available for p in [\"ear_left\", \"ear_right\"]):\n        ear_d = np.sqrt(\n            (single_mouse[\"ear_left\"][\"x\"] - single_mouse[\"ear_right\"][\"x\"])**2 +\n            (single_mouse[\"ear_left\"][\"y\"] - single_mouse[\"ear_right\"][\"y\"])**2\n        )\n        for off in [-30, -20, -10, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f\"ear_o{off}\"] = ear_d.shift(-o)\n        w = _scale(30, fps)\n        X[\"ear_con\"] = ear_d.rolling(w, min_periods=1, center=True).std() / \\\n                       (ear_d.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n    return X.astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.885812Z","iopub.execute_input":"2025-12-13T09:40:13.886098Z","iopub.status.idle":"2025-12-13T09:40:13.917619Z","shell.execute_reply.started":"2025-12-13T09:40:13.886077Z","shell.execute_reply":"2025-12-13T09:40:13.916862Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Pair-mouse features","metadata":{}},{"cell_type":"code","source":"def add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n    if \"body_center\" not in avail_A or \"body_center\" not in avail_B:\n        return X\n\n    rel_x = mouse_pair[\"A\"][\"body_center\"][\"x\"] - mouse_pair[\"B\"][\"body_center\"][\"x\"]\n    rel_y = mouse_pair[\"A\"][\"body_center\"][\"y\"] - mouse_pair[\"B\"][\"body_center\"][\"y\"]\n    rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n    A_vx = mouse_pair[\"A\"][\"body_center\"][\"x\"].diff()\n    A_vy = mouse_pair[\"A\"][\"body_center\"][\"y\"].diff()\n    B_vx = mouse_pair[\"B\"][\"body_center\"][\"x\"].diff()\n    B_vy = mouse_pair[\"B\"][\"body_center\"][\"y\"].diff()\n\n    A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n    B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n    for window in [30, 60]:\n        ws = _scale(window, fps)\n        X[f\"A_ld{window}\"] = A_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n        X[f\"B_ld{window}\"] = B_lead.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    approach = -rel_dist.diff()\n    chase = approach * B_lead\n    w = 30\n    ws = _scale(w, fps)\n    X[f\"chase_{w}\"] = chase.rolling(ws, min_periods=max(1, ws // 6)).mean()\n\n    for window in [60, 120]:\n        ws = _scale(window, fps)\n        A_sp = np.sqrt(A_vx**2 + A_vy**2)\n        B_sp = np.sqrt(B_vx**2 + B_vy**2)\n        X[f\"sp_cor{window}\"] = A_sp.rolling(ws, min_periods=max(1, ws // 6)).corr(B_sp)\n\n    return X\n\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n    \"\"\"\n    Feature cho cặp chuột (pair).\n    \"\"\"\n    avail_A = mouse_pair[\"A\"].columns.get_level_values(0)\n    avail_B = mouse_pair[\"B\"].columns.get_level_values(0)\n\n    X = pd.DataFrame({\n        f\"12+{p1}+{p2}\": np.square(mouse_pair[\"A\"][p1] - mouse_pair[\"B\"][p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n        if p1 in avail_A and p2 in avail_B\n    })\n    X = X.reindex(\n        columns=[f\"12+{p1}+{p2}\" for p1, p2 in itertools.product(body_parts_tracked, repeat=2)],\n        copy=False\n    ) #reindex đồng nhất schema feature giữa các video(đủ cột + đúng thứ tự) để model training không bị lệch cột hoặc lỗi thiếu feature\n\n    # Thêm một số feature interaction\n    if (\"A\", \"ear_left\") in mouse_pair.columns and (\"B\", \"ear_left\") in mouse_pair.columns:\n        lag = _scale(10, fps)\n        shA = mouse_pair[\"A\"][\"ear_left\"].shift(lag)\n        shB = mouse_pair[\"B\"][\"ear_left\"].shift(lag)\n        speeds = pd.DataFrame({\n            \"sp_A\": np.square(mouse_pair[\"A\"][\"ear_left\"] - shA).sum(axis=1, skipna=False),\n            \"sp_AB\": np.square(mouse_pair[\"A\"][\"ear_left\"] - shB).sum(axis=1, skipna=False),\n            \"sp_B\": np.square(mouse_pair[\"B\"][\"ear_left\"] - shB).sum(axis=1, skipna=False),\n        })\n        X = pd.concat([X, speeds], axis=1)\n        \n    if all(p in avail_A for p in [\"nose\", \"tail_base\"]) and all(p in avail_B for p in [\"nose\", \"tail_base\"]):\n        dir_A = mouse_pair[\"A\"][\"nose\"] - mouse_pair[\"A\"][\"tail_base\"]\n        dir_B = mouse_pair[\"B\"][\"nose\"] - mouse_pair[\"B\"][\"tail_base\"]\n        X[\"rel_ori\"] = (dir_A[\"x\"] * dir_B[\"x\"] + dir_A[\"y\"] * dir_B[\"y\"]) / (\n            np.sqrt(dir_A[\"x\"]**2 + dir_A[\"y\"]**2) *\n            np.sqrt(dir_B[\"x\"]**2 + dir_B[\"y\"]**2) + 1e-6\n        )\n\n    if all(p in avail_A for p in [\"nose\"]) and all(p in avail_B for p in [\"nose\"]):\n        cur = np.square(mouse_pair[\"A\"][\"nose\"] - mouse_pair[\"B\"][\"nose\"]).sum(axis=1, skipna=False)\n        lag = _scale(10, fps)\n        shA_n = mouse_pair[\"A\"][\"nose\"].shift(lag)\n        shB_n = mouse_pair[\"B\"][\"nose\"].shift(lag)\n        past = np.square(shA_n - shB_n).sum(axis=1, skipna=False)\n        X[\"appr\"] = cur - past\n\n    if \"body_center\" in avail_A and \"body_center\" in avail_B:\n        cd = np.sqrt(\n            (mouse_pair[\"A\"][\"body_center\"][\"x\"] - mouse_pair[\"B\"][\"body_center\"][\"x\"])**2 +\n            (mouse_pair[\"A\"][\"body_center\"][\"y\"] - mouse_pair[\"B\"][\"body_center\"][\"y\"])**2\n        )\n        X[\"v_cls\"] = (cd < 5.0).astype(float)\n        X[\"cls\"] = ((cd >= 5.0) & (cd < 15.0)).astype(float)\n        X[\"med\"] = ((cd >= 15.0) & (cd < 30.0)).astype(float)\n        X[\"far\"] = (cd >= 30.0).astype(float)\n\n        cd_full = np.square(mouse_pair[\"A\"][\"body_center\"] - mouse_pair[\"B\"][\"body_center\"]).sum(axis=1, skipna=False)\n\n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f\"d_m{w}\"] = cd_full.rolling(ws, **roll).mean()\n            X[f\"d_s{w}\"] = cd_full.rolling(ws, **roll).std()\n            X[f\"d_mn{w}\"] = cd_full.rolling(ws, **roll).min()\n            X[f\"d_mx{w}\"] = cd_full.rolling(ws, **roll).max()\n\n            d_var = cd_full.rolling(ws, **roll).var()\n            X[f\"int{w}\"] = 1 / (1 + d_var)\n\n            Axd = mouse_pair[\"A\"][\"body_center\"][\"x\"].diff()\n            Ayd = mouse_pair[\"A\"][\"body_center\"][\"y\"].diff()\n            Bxd = mouse_pair[\"B\"][\"body_center\"][\"x\"].diff()\n            Byd = mouse_pair[\"B\"][\"body_center\"][\"y\"].diff()\n            coord = Axd * Bxd + Ayd * Byd\n            X[f\"co_m{w}\"] = coord.rolling(ws, **roll).mean()\n            X[f\"co_s{w}\"] = coord.rolling(ws, **roll).std()\n\n    if \"nose\" in avail_A and \"nose\" in avail_B:\n        nn = np.sqrt(\n            (mouse_pair[\"A\"][\"nose\"][\"x\"] - mouse_pair[\"B\"][\"nose\"][\"x\"])**2 +\n            (mouse_pair[\"A\"][\"nose\"][\"y\"] - mouse_pair[\"B\"][\"nose\"][\"y\"])**2\n        )\n        for lag in [10, 20, 40]:\n            l = _scale(lag, fps)\n            X[f\"nn_lg{lag}\"] = nn.shift(l)\n            X[f\"nn_ch{lag}\"] = nn - nn.shift(l)\n            is_cl = (nn < 10.0).astype(float)\n            X[f\"cl_ps{lag}\"] = is_cl.rolling(l, min_periods=1).mean()\n\n    if \"body_center\" in avail_A and \"body_center\" in avail_B:\n        Avx = mouse_pair[\"A\"][\"body_center\"][\"x\"].diff()\n        Avy = mouse_pair[\"A\"][\"body_center\"][\"y\"].diff()\n        Bvx = mouse_pair[\"B\"][\"body_center\"][\"x\"].diff()\n        Bvy = mouse_pair[\"B\"][\"body_center\"][\"y\"].diff()\n        val = (Avx * Bvx + Avy * Bvy) / (\n            np.sqrt(Avx**2 + Avy**2) * np.sqrt(Bvx**2 + Bvy**2) + 1e-6\n        )\n\n        for off in [-30, -20, -10, 0, 10, 20, 30]:\n            o = _scale_signed(off, fps)\n            X[f\"va_{off}\"] = val.shift(-o)\n\n        w = _scale(30, fps)\n        cd_full = np.square(mouse_pair[\"A\"][\"body_center\"] - mouse_pair[\"B\"][\"body_center\"]).sum(axis=1, skipna=False)\n        X[\"int_con\"] = cd_full.rolling(w, min_periods=1, center=True).std() / \\\n                       (cd_full.rolling(w, min_periods=1, center=True).mean() + 1e-6)\n\n        X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n    return X.astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.919416Z","iopub.execute_input":"2025-12-13T09:40:13.919850Z","iopub.status.idle":"2025-12-13T09:40:13.949510Z","shell.execute_reply.started":"2025-12-13T09:40:13.919821Z","shell.execute_reply":"2025-12-13T09:40:13.948744Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train/Validate","metadata":{}},{"cell_type":"code","source":"def simple_tune_threshold(oof, y):\n    best_t, best_f1 = 0.5, -1\n    for t in np.linspace(0.0, 1.0, 51):\n        f1 = f1_score(y, (oof >= t), zero_division=0)\n        if f1 > best_f1:\n            best_f1 = f1\n            best_t = t\n    return best_t, best_f1\n\n\ndef cross_validate_classifier(X, label, meta, body_parts_tracked_str, section, mode_key):\n    os.makedirs(f\"{MABe.model_subdir}/{section}\", exist_ok=True)\n\n    oof_df = pd.DataFrame(index=meta.video_frame)\n    f1_list = []\n    thresholds = {}\n    submission_list = []\n\n    for action in label.columns:\n        mask = ~label[action].isna().values\n        y_action = label[action][mask].values.astype(int)\n        X_action = X[mask]\n        groups = meta.video_id[mask]\n\n        if len(np.unique(groups)) < 2:\n            continue\n        if (y_action == 0).all():\n            oof_action = np.zeros(len(y_action))\n            thresholds[action] = 0.5\n        else:\n            try:\n                cv = GroupKFold(n_splits=min(2, len(np.unique(groups))))\n                oof_action = np.zeros(len(y_action))\n                models = []\n\n                for tr_idx, va_idx in cv.split(X_action, y_action, groups):\n                    model = LGBMClassifier(**MABe.lgb_params)\n                    model.fit(X_action.iloc[tr_idx], y_action[tr_idx])\n                    oof_action[va_idx] = model.predict_proba(X_action.iloc[va_idx])[:, 1]\n                    models.append(model)\n\n                thr, f1_val = simple_tune_threshold(oof_action, y_action)\n                thresholds[action] = thr\n                f1_list.append((body_parts_tracked_str, mode_key, action, f1_val, thr))\n                print(f\"\\t[{mode_key}] Action={action}, F1={f1_val:.4f}, thr={thr:.3f}\")\n\n                # Lưu model\n                model_dir = f\"{MABe.model_subdir}/{section}/{action}\"\n                os.makedirs(model_dir, exist_ok=True)\n                joblib.dump(models, f\"{model_dir}/lgb_trainer.pkl\")\n            except Exception as e:\n                print(f\"\\tFailed train action {action}: {e}\")\n                oof_action = np.zeros(len(y_action))\n                thresholds[action] = 0.5\n\n        tmp = np.zeros(len(label))\n        tmp[mask] = oof_action\n        oof_df[action] = tmp\n\n        del oof_action, mask, X_action, y_action, groups\n        gc.collect()\n\n    sub_part = predict_multiclass(oof_df, meta, thresholds)\n    submission_list.append(sub_part)\n    return submission_list, f1_list, thresholds","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.950594Z","iopub.execute_input":"2025-12-13T09:40:13.950836Z","iopub.status.idle":"2025-12-13T09:40:13.980680Z","shell.execute_reply.started":"2025-12-13T09:40:13.950812Z","shell.execute_reply":"2025-12-13T09:40:13.979699Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submit\n","metadata":{}},{"cell_type":"code","source":"def predict_multiclass(pred, meta, thresholds, min_duration=3, smoothing_window=5):\n    if pred.empty:\n        return pd.DataFrame()\n\n    required_cols = {\"video_id\", \"agent_id\", \"target_id\", \"video_frame\"}\n    if not required_cols.issubset(set(meta.columns)):\n        print(\"\\t    Meta missing required columns\")\n        return pd.DataFrame()\n\n    # Áp dụng temporal smoothing\n    if smoothing_window > 1 and len(pred) >= smoothing_window:\n        pred_smooth = pred.rolling(\n            window=smoothing_window, \n            center=True, \n            min_periods=1\n        ).mean()\n    else:\n        pred_smooth = pred\n\n    # Chọn action có prob max\n    ama = np.argmax(pred_smooth.values, axis=1)\n    max_proba = pred_smooth.max(axis=1).values\n\n    threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n    action_thresholds = threshold_array[ama]\n    ama = np.where(max_proba >= action_thresholds, ama, -1)\n\n    # Áp dụng majority voting trong window nhỏ để giảm noise\n    voting_window = 3\n    if len(ama) >= voting_window:\n        ama_smooth = pd.Series(ama).rolling(\n            window=voting_window, \n            center=True, \n            min_periods=1\n        ).apply(lambda x: pd.Series(x).mode()[0] if len(pd.Series(x).mode()) > 0 else x.iloc[0])\n        ama = ama_smooth.values.astype(int)\n\n    submission_parts = []\n    if len(ama) == 0:\n        return pd.DataFrame()\n\n    current_action = ama[0]\n    start_frame = meta.video_frame.iloc[0]\n    duration = 1\n\n    for i in range(1, len(ama)):\n        if ama[i] == current_action:\n            duration += 1\n            continue\n\n        if current_action >= 0 and duration >= min_duration:\n            try:\n                video_id = meta.video_id.iloc[i - 1]\n                agent_id = meta.agent_id.iloc[i - 1]\n                target_id = meta.target_id.iloc[i - 1]\n                stop_frame = meta.video_frame.iloc[i - 1] + 1\n                submission_parts.append({\n                    \"video_id\": video_id,\n                    \"agent_id\": agent_id,\n                    \"target_id\": target_id,\n                    \"action\": pred.columns[current_action],\n                    \"start_frame\": start_frame,\n                    \"stop_frame\": stop_frame\n                })\n            except Exception as e:\n                print(f\"\\t      Error creating segment: {e}\")\n\n        current_action = ama[i]\n        start_frame = meta.video_frame.iloc[i]\n        duration = 1\n\n    # Đoạn cuối\n    if current_action >= 0 and duration >= min_duration:\n        try:\n            video_id = meta.video_id.iloc[-1]\n            agent_id = meta.agent_id.iloc[-1]\n            target_id = meta.target_id.iloc[-1]\n            stop_frame = meta.video_frame.iloc[-1] + 1\n            submission_parts.append({\n                \"video_id\": video_id,\n                \"agent_id\": agent_id,\n                \"target_id\": target_id,\n                \"action\": pred.columns[current_action],\n                \"start_frame\": start_frame,\n                \"stop_frame\": stop_frame\n            })\n        except Exception as e:\n            print(f\"\\t      Error creating last segment: {e}\")\n\n    return pd.DataFrame(submission_parts)\n\ndef predict_with_ensemble_weights(X_te, models, action, weight_strategy=\"uniform\"):\n    \"\"\"\n    Dự đoán với weighted ensemble\n    weight_strategy: \"uniform\", \"inverse_fold\", \"performance_based\"\n    \"\"\"\n    probas = []\n    for mdl in models:\n        prob = mdl.predict_proba(X_te)[:, 1]\n        probas.append(prob)\n    \n    if weight_strategy == \"uniform\":\n        return np.mean(probas, axis=0)\n    elif weight_strategy == \"inverse_fold\":\n        weights = np.arange(1, len(probas) + 1)\n        weights = weights / weights.sum()\n        return np.average(probas, axis=0, weights=weights)\n    else:\n        return np.mean(probas, axis=0)\n\ndef load_thresholds_for_submit():\n    \"\"\"\n    Tìm thresholds.pkl trong:\n        /kaggle/input/kk-lightgbm/thresholds.pkl\n        /kaggle/input/kk-lightgbm/lightgbm/thresholds.pkl\n    \"\"\"\n    candidates = [\n        os.path.join(MABe.model_root, \"thresholds.pkl\"),\n        os.path.join(MABe.model_root, MABe.model_subdir, \"thresholds.pkl\"),\n    ]\n    for p in candidates:\n        if os.path.exists(p):\n            print(f\"Loaded thresholds from: {p}\")\n            return joblib.load(p)\n\n    print(\"WARNING: thresholds.pkl not found, using default 0.5\")\n    return {\n        \"single\": {\"default\": 0.5},\n        \"pair\": {\"default\": 0.5}\n    }\n\ndef submit(body_parts_tracked_str, switch_tr, section, thresholds_section):\n    body_parts_tracked = json.loads(body_parts_tracked_str)\n    if len(body_parts_tracked) > 5:\n        body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n\n    test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n    print(f\"\\tTest subset size: {len(test_subset)}\")\n\n    if len(test_subset) == 0:\n        print(\"\\tNo test videos for this body_parts_tracked.\")\n        return []\n\n    fps_lookup = (\n        test_subset[[\"video_id\", \"frames_per_second\"]]\n        .drop_duplicates(\"video_id\")\n        .set_index(\"video_id\")[\"frames_per_second\"]\n        .to_dict()\n    )\n\n    models_cache = {}\n    all_actions = set()\n\n    for _, row in test_subset.iterrows():\n        if type(row.behaviors_labeled) == str:\n            try:\n                behaviors = json.loads(row.behaviors_labeled)\n                for b in behaviors:\n                    if isinstance(b, str) and \",\" in b:\n                        parts = b.split(\",\")\n                        action = parts[2] if len(parts) >= 3 else b\n                    else:\n                        action = str(b)\n                    all_actions.add(action)\n            except Exception as e:\n                print(f\"\\tFailed to parse behaviors: {e}\")\n\n    print(f\"\\tWill try to load models for {len(all_actions)} actions: {list(all_actions)}\")\n\n    loaded_actions = 0\n    for action in all_actions:\n        try:\n            pattern1 = os.path.join(MABe.model_root, MABe.model_subdir, str(section), action, \"lgb_trainer.pkl\")\n            pattern2 = os.path.join(MABe.model_root, str(section), action, \"lgb_trainer.pkl\")\n\n            model_files = glob.glob(pattern1)\n            if not model_files:\n                model_files = glob.glob(pattern2)\n\n            if model_files:\n                models = joblib.load(model_files[0])\n                models_cache[action] = models\n                loaded_actions += 1\n                print(f\"\\t✓ Loaded model for action '{action}' from {model_files[0]}\")\n            else:\n                print(f\"\\t✗ Model not found for action '{action}'\")\n        except Exception as e:\n            print(f\"\\t✗ Failed to load model for action '{action}': {e}\")\n\n    print(f\"\\tLoaded {loaded_actions}/{len(all_actions)} actions.\")\n    if loaded_actions == 0:\n        print(\"\\tNo models loaded, skip this section.\")\n        return []\n\n    def process_single_video(row):\n        try:\n            lab_id = row[\"lab_id\"]\n            video_id = row[\"video_id\"]\n            print(f\"\\tProcessing video: {video_id}\")\n\n            video_data = pd.DataFrame([row])\n            gen = generate_mouse_data(\n                video_data,\n                \"test\",\n                traintest_directory=MABe.test_tracking_path,\n                generate_single=(switch_tr == \"single\"),\n                generate_pair=(switch_tr == \"pair\")\n            )\n\n            video_submissions = []\n            count_items = 0\n\n            for sw_te, data_te, meta_te, actions_te in gen:\n                count_items += 1\n                print(f\"\\t  Part: {sw_te}, actions: {list(actions_te)}\")\n\n                if \"video_frame\" not in meta_te.columns:\n                    print(\"\\t  meta_te missing video_frame\")\n                    continue\n\n                fps_i = _fps_from_meta(meta_te, fps_lookup, default_fps=30.0)\n                if sw_te == \"single\":\n                    X_te = transform_single(data_te, body_parts_tracked, fps_i).astype(np.float32)\n                else:\n                    X_te = transform_pair(data_te, body_parts_tracked, fps_i).astype(np.float32)\n\n                del data_te\n                gc.collect()\n\n                if X_te.shape[0] == 0:\n                    print(\"\\t  Warning: empty feature matrix\")\n                    continue\n\n                pred = pd.DataFrame(index=meta_te.video_frame)\n                predicted_actions = 0\n\n                for action in actions_te:\n                    if action not in models_cache:\n                        continue\n                    try:\n                        models = models_cache[action]\n                        mean_prob = predict_with_ensemble_weights(\n                            X_te, models, action, weight_strategy=\"uniform\"\n                        )\n                        pred[action] = mean_prob\n                        predicted_actions += 1\n                        thr = thresholds_section.get(action, 0.27)\n                        print(f\"\\t    {action}: mean={mean_prob.mean():.4f}, max={mean_prob.max():.4f}, thr={thr}\")\n                    except Exception as e:\n                        print(f\"\\t    Failed to predict action '{action}': {e}\")\n\n                del X_te\n                gc.collect()\n\n                print(f\"\\t    Predicted {predicted_actions}/{len(actions_te)} actions.\")\n\n                if not pred.empty and pred.shape[1] > 0:\n                    # SỬ DỤNG PREDICT CÓ SMOOTHING\n                    part_sub = predict_multiclass(\n                        pred, meta_te, thresholds_section, \n                        min_duration=2, smoothing_window=5\n                    )\n                    if len(part_sub) > 0:\n                        video_submissions.append(part_sub)\n                        print(f\"\\t    Got {len(part_sub)} segments from this part.\")\n                    else:\n                        print(\"\\t    All segments filtered by thresholds.\")\n                else:\n                    print(\"\\t    No valid predictions for this part.\")\n\n                del pred\n                gc.collect()\n\n            print(f\"\\tVideo {video_id} done: {len(video_submissions)} partial submissions, {count_items} generator items.\")\n            return video_submissions\n        except Exception as e:\n            print(f\"\\tError processing video {row['video_id']}: {e}\")\n            return []\n\n    submission_list = []\n    print(f\"\\tStart processing {len(test_subset)} videos for switch_tr={switch_tr} ...\")\n    for idx, row in test_subset.iterrows():\n        print(f\"\\n\\t=== Video {idx + 1}/{len(test_subset)} ===\")\n        result = process_single_video(row)\n        if result:\n            submission_list.extend(result)\n        if idx % 5 == 0:\n            gc.collect()\n\n    del models_cache\n    gc.collect()\n\n    print(f\"\\tFinished section={section}, switch_tr={switch_tr}, got {len(submission_list)} partial submissions.\")\n    return submission_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:13.981674Z","iopub.execute_input":"2025-12-13T09:40:13.981935Z","iopub.status.idle":"2025-12-13T09:40:14.013384Z","shell.execute_reply.started":"2025-12-13T09:40:13.981910Z","shell.execute_reply":"2025-12-13T09:40:14.012674Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hậu xử lý submission","metadata":{}},{"cell_type":"code","source":"def post_process_segments(submission_df, min_gap=5, min_duration=3):\n    \"\"\"\n    Merge các segments gần nhau và loại bỏ segments quá ngắn\n    min_gap: số frame tối thiểu giữa 2 segments để coi là riêng biệt\n    min_duration: độ dài tối thiểu của segment\n    \"\"\"\n    if len(submission_df) == 0:\n        return submission_df\n    \n    processed_list = []\n    \n    for (video_id, agent_id, target_id, action), group in submission_df.groupby(\n        [\"video_id\", \"agent_id\", \"target_id\", \"action\"]\n    ):\n        group = group.sort_values(\"start_frame\").reset_index(drop=True)\n        \n        if len(group) == 0:\n            continue\n            \n        merged = []\n        current_start = group.iloc[0][\"start_frame\"]\n        current_stop = group.iloc[0][\"stop_frame\"]\n        \n        for i in range(1, len(group)):\n            row = group.iloc[i]\n            # Nếu gap nhỏ hơn threshold thì merge\n            if row[\"start_frame\"] - current_stop <= min_gap:\n                current_stop = max(current_stop, row[\"stop_frame\"])\n            else:\n                # Lưu segment hiện tại nếu đủ dài\n                if current_stop - current_start >= min_duration:\n                    merged.append({\n                        \"video_id\": video_id,\n                        \"agent_id\": agent_id,\n                        \"target_id\": target_id,\n                        \"action\": action,\n                        \"start_frame\": current_start,\n                        \"stop_frame\": current_stop\n                    })\n                current_start = row[\"start_frame\"]\n                current_stop = row[\"stop_frame\"]\n        \n        # Xử lý segment cuối cùng\n        if current_stop - current_start >= min_duration:\n            merged.append({\n                \"video_id\": video_id,\n                \"agent_id\": agent_id,\n                \"target_id\": target_id,\n                \"action\": action,\n                \"start_frame\": current_start,\n                \"stop_frame\": current_stop\n            })\n        \n        processed_list.extend(merged)\n    \n    return pd.DataFrame(processed_list)\n\ndef robustify(submission, dataset, traintest, traintest_directory=None):\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n    old_submission = submission.copy()\n\n    # Đảm bảo start/stop là int\n    submission[\"start_frame\"] = pd.to_numeric(\n        submission[\"start_frame\"], errors=\"coerce\"\n    ).fillna(0).astype(int)\n    submission[\"stop_frame\"] = pd.to_numeric(\n        submission[\"stop_frame\"], errors=\"coerce\"\n    ).fillna(0).astype(int)\n\n    # Bỏ dòng start >= stop\n    submission = submission[submission.start_frame < submission.stop_frame]\n    if len(submission) != len(old_submission):\n        print(\"NOTE: Dropped rows with start_frame >= stop_frame\")\n\n    old_submission = submission.copy()\n    group_list = []\n\n    # Xử lý trùng lặp theo (video, agent, target)\n    for _, group in submission.groupby([\"video_id\", \"agent_id\", \"target_id\"]):\n        group = group.sort_values(\"start_frame\")\n        mask = np.ones(len(group), dtype=bool)\n        last_stop = -1\n        for i, (_, row) in enumerate(group.iterrows()):\n            if row[\"start_frame\"] < last_stop:\n                mask[i] = False\n            else:\n                last_stop = row[\"stop_frame\"]\n        group_list.append(group[mask])\n\n    submission = pd.concat(group_list)\n    if len(submission) != len(old_submission):\n        print(\"NOTE: Dropped overlapping segments\")\n\n    # Điền thêm video không có prediction\n    s_list = []\n    for _, row in dataset.iterrows():\n        lab_id = row[\"lab_id\"]\n        if lab_id.startswith(\"MABe22\"):\n            continue\n\n        video_id = row[\"video_id\"]\n        if (submission.video_id == video_id).any():\n            continue\n        if type(row.behaviors_labeled) != str:\n            continue\n\n        print(f\"Video {video_id} has no predictions -> filling dummy segments\")\n\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n\n        vid_behaviors = json.loads(row[\"behaviors_labeled\"])\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(\",\") for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=[\"agent\", \"target\", \"action\"])\n\n        start_frame = vid.video_frame.min()\n        stop_frame = vid.video_frame.max() + 1\n\n        for (agent, target), actions in vid_behaviors.groupby([\"agent\", \"target\"]):\n            batch_length = int(np.ceil((stop_frame - start_frame) / len(actions)))\n            for i, (_, ar) in enumerate(actions.iterrows()):\n                b_start = start_frame + i * batch_length\n                b_stop = min(b_start + batch_length, stop_frame)\n                s_list.append((video_id, agent, target, ar[\"action\"], b_start, b_stop))\n\n    if len(s_list) > 0:\n        df_fill = pd.DataFrame(\n            s_list,\n            columns=[\"video_id\", \"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"]\n        )\n        submission = pd.concat([submission, df_fill], ignore_index=True)\n        print(\"NOTE: Filled empty videos with dummy segments\")\n\n    submission = submission.reset_index(drop=True)\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:14.014462Z","iopub.execute_input":"2025-12-13T09:40:14.014785Z","iopub.status.idle":"2025-12-13T09:40:14.041503Z","shell.execute_reply.started":"2025-12-13T09:40:14.014759Z","shell.execute_reply":"2025-12-13T09:40:14.040718Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Main\n","metadata":{}},{"cell_type":"code","source":"submission_list = []\nf1_list = []\n\nif MABe.mode == \"validate\":\n    thresholds_all = {\"single\": {}, \"pair\": {}}\n\n    for section in range(1, len(body_parts_tracked_list)):\n        body_parts_tracked_str = body_parts_tracked_list[section]\n        try:\n            body_parts_tracked = json.loads(body_parts_tracked_str)\n            print(f\"\\n[{section}/{len(body_parts_tracked_list)-1}] body_parts_tracked = {body_parts_tracked}\")\n\n            if len(body_parts_tracked) > 5:\n                body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n\n            train_subset = train[train.body_parts_tracked == body_parts_tracked_str]\n            if len(train_subset) == 0:\n                print(\"\\tNo train videos for this subset.\")\n                continue\n\n            fps_lookup = (\n                train_subset[[\"video_id\", \"frames_per_second\"]]\n                .drop_duplicates(\"video_id\")\n                .set_index(\"video_id\")[\"frames_per_second\"]\n                .to_dict()\n            )\n\n            single_data, single_meta, single_label = [], [], []\n            pair_data, pair_meta, pair_label = [], [], []\n\n            # Sinh dữ liệu từ train\n            for sw, data, meta, label in generate_mouse_data(train_subset, \"train\"):\n                if sw == \"single\":\n                    single_data.append(data)\n                    single_meta.append(meta)\n                    single_label.append(label)\n                else:\n                    pair_data.append(data)\n                    pair_meta.append(meta)\n                    pair_label.append(label)\n                del data, meta, label\n            gc.collect()\n\n            # -------- TRAIN CHO SINGLE --------\n            if len(single_data) > 0:\n                feats = []\n                for d, m in zip(single_data, single_meta):\n                    fps_i = _fps_from_meta(m, fps_lookup, default_fps=30.0)\n                    X_i = transform_single(d, body_parts_tracked, fps_i).astype(np.float32)\n                    feats.append(X_i)\n                    del X_i, fps_i\n                gc.collect()\n\n                X_tr = pd.concat(feats, axis=0, ignore_index=True)\n                y_tr = pd.concat(single_label, axis=0, ignore_index=True)\n                meta_tr = pd.concat(single_meta, axis=0, ignore_index=True)\n\n                tmp_sub_list, tmp_f1, tmp_thr = cross_validate_classifier(\n                    X_tr, y_tr, meta_tr, body_parts_tracked_str, section, mode_key=\"single\"\n                )\n                thresholds_all[\"single\"][str(section)] = tmp_thr\n                f1_list.extend(tmp_f1)\n                submission_list.extend(tmp_sub_list)\n                del feats, single_data, single_label, single_meta, X_tr, y_tr, meta_tr\n                gc.collect()\n\n            # -------- TRAIN CHO PAIR --------\n            if len(pair_data) > 0:\n                feats = []\n                for d, m in zip(pair_data, pair_meta):\n                    fps_i = _fps_from_meta(m, fps_lookup, default_fps=30.0)\n                    X_i = transform_pair(d, body_parts_tracked, fps_i).astype(np.float32)\n                    feats.append(X_i)\n                    del X_i, fps_i\n                gc.collect()\n\n                X_tr = pd.concat(feats, axis=0, ignore_index=True)\n                y_tr = pd.concat(pair_label, axis=0, ignore_index=True)\n                meta_tr = pd.concat(pair_meta, axis=0, ignore_index=True)\n\n                tmp_sub_list, tmp_f1, tmp_thr = cross_validate_classifier(\n                    X_tr, y_tr, meta_tr, body_parts_tracked_str, section, mode_key=\"pair\"\n                )\n                thresholds_all[\"pair\"][str(section)] = tmp_thr\n                f1_list.extend(tmp_f1)\n                submission_list.extend(tmp_sub_list)\n                del feats, pair_data, pair_label, pair_meta, X_tr, y_tr, meta_tr\n                gc.collect()\n\n        except Exception as e:\n            print(f\"\\tError in section {section}: {e}\")\n\n    if len(f1_list) > 0:\n        f1_df = pd.DataFrame(\n            f1_list,\n            columns=[\"body_parts_tracked_str\", \"mode\", \"action\", \"binary_F1\", \"threshold\"]\n        )\n        print(f\"\\nMean F1 (binary): {f1_df['binary_F1'].mean():.4f}\")\n    else:\n        f1_df = pd.DataFrame(columns=[\"body_parts_tracked_str\", \"mode\", \"action\", \"binary_F1\", \"threshold\"])\n        print(\"No F1 data collected.\")\n\n    os.makedirs(MABe.model_subdir, exist_ok=True)\n    joblib.dump(thresholds_all, os.path.join(MABe.model_subdir, \"thresholds.pkl\"))\n    joblib.dump(f1_df, os.path.join(MABe.model_subdir, \"scores.pkl\"))\n    print(f\"\\nSaved thresholds & scores into folder: {MABe.model_subdir}/\")\n\nelif MABe.mode == \"submit\":\n    # Dùng model + threshold đã có sẵn trong /kaggle/input/kk-lightgbm\n    thresholds_all = load_thresholds_for_submit()\n\n    for section in range(1, len(body_parts_tracked_list)):\n        body_parts_tracked_str = body_parts_tracked_list[section]\n        try:\n            body_parts_tracked = json.loads(body_parts_tracked_str)\n        except Exception:\n            continue\n\n        print(f\"\\n[{section}/{len(body_parts_tracked_list)-1}] body_parts_tracked = {body_parts_tracked}\")\n\n        # single\n        thr_single = thresholds_all.get(\"single\", {}).get(str(section), {})\n        if thr_single:\n            submission_list.extend(\n                submit(body_parts_tracked_str, \"single\", section, thr_single)\n            )\n\n        # pair\n        thr_pair = thresholds_all.get(\"pair\", {}).get(str(section), {})\n        if thr_pair:\n            submission_list.extend(\n                submit(body_parts_tracked_str, \"pair\", section, thr_pair)\n            )\n\n    if len(submission_list) > 0:\n        submission = pd.concat(submission_list, ignore_index=True)\n        print(\"\\nApplying post-processing...\")\n        submission = post_process_segments(submission, min_gap=5, min_duration=2)\n        print(f\"After post-processing: {len(submission)} segments\")\n    else:\n        # Fallback: 1 dòng dummy cho chắc\n        submission = pd.DataFrame({\n            \"video_id\": [test.video_id.iloc[0]],\n            \"agent_id\": [\"mouse1\"],\n            \"target_id\": [\"self\"],\n            \"action\": [\"rear\"],\n            \"start_frame\": [0],\n            \"stop_frame\": [100]\n        })\n\n    submission_robust = robustify(submission, test, \"test\")\n    submission_robust.index.name = \"row_id\"\n    submission_robust.to_csv(\"submission.csv\", index=True)\n    print(f\"\\nSaved submission.csv, shape = {submission_robust.shape}\")\n    print(submission_robust.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-13T09:40:14.042369Z","iopub.execute_input":"2025-12-13T09:40:14.042747Z","execution_failed":"2025-12-13T09:41:49.419Z"}},"outputs":[],"execution_count":null}]}