{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14127085,"sourceType":"datasetVersion","datasetId":9001058},{"sourceId":14147190,"sourceType":"datasetVersion","datasetId":9016284},{"sourceId":279806245,"sourceType":"kernelVersion"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport polars as pl\nimport numpy as np\nimport json\nfrom tqdm import tqdm\nimport itertools\nimport gc\nfrom sklearn.model_selection import StratifiedKFold\nimport optuna\nfrom sklearn.metrics import f1_score\nimport pickle\nimport torch\nimport joblib\nimport gc\nimport polars as pl\nfrom collections import defaultdict\nfrom pathlib import Path\nimport re\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier\nimport lightgbm as lgb\nimport os\nfrom joblib import Parallel, delayed","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:57.951915Z","iopub.status.busy":"2025-12-12T17:46:57.950975Z","iopub.status.idle":"2025-12-12T17:46:57.957846Z","shell.execute_reply":"2025-12-12T17:46:57.956879Z","shell.execute_reply.started":"2025-12-12T17:46:57.951888Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BODY_PARTS = [\n    \"ear_left\",\n    \"ear_right\",\n    \"nose\",\n    \"neck\",\n    \"body_center\",\n    \"lateral_left\",\n    \"lateral_right\",\n    \"hip_left\",\n    \"hip_right\",\n    \"tail_base\",\n    \"tail_tip\",\n]\n\nSELF_ACTIONS = [\n    \"biteobject\",\n    \"climb\",\n    \"dig\",\n    \"exploreobject\",\n    \"freeze\",\n    \"genitalgroom\",\n    \"huddle\",\n    \"rear\",\n    \"rest\",\n    \"run\",\n    \"selfgroom\",\n]\n\nPAIR_ACTIONS = [\n    \"allogroom\",\n    \"approach\",\n    \"attack\",\n    \"attemptmount\",\n    \"avoid\",\n    \"chase\",\n    \"chaseattack\",\n    \"defend\",\n    \"disengage\",\n    \"dominance\",\n    \"dominancegroom\",\n    \"dominancemount\",\n    \"ejaculate\",\n    \"escape\",\n    \"flinch\",\n    \"follow\",\n    \"intromit\",\n    \"mount\",\n    \"reciprocalsniff\",\n    \"shepherd\",\n    \"sniff\",\n    \"sniffbody\",\n    \"sniffface\",\n    \"sniffgenital\",\n    \"submit\",\n    \"tussle\",\n]\nINDEX_COLS = [\n    \"video_id\",\n    \"agent_mouse_id\",\n    \"target_mouse_id\",\n    \"video_frame\",\n]\n\nMODEL_CONFIG = {\n    'l': {\n        'n_estimators': 200,  \n        'learning_rate': 0.05,\n        'max_depth': 7,\n        'num_leaves': 31,\n        'min_child_samples': 40,\n        'min_data_in_bin': 3,\n        'min_sum_hessian_in_leaf': 1e-3,\n        'feature_fraction': 0.8,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 1,\n        'reg_alpha': 1,\n        'reg_lambda': 2.0,\n        'max_bin': 255,\n        'device': 'cpu',\n        # 'gpu_use_dp': False,\n        'force_col_wise': True,\n        'zero_as_missing': False,\n        'objective': 'binary',\n        'metric': 'binary_logloss',\n        'verbose': -1,\n        'random_state': 42,\n        'n_jobs': -1\n    }\n\n}\n\n# TRAINING | SUBMIT\nSTATUS = 'SUBMIT'","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:57.959833Z","iopub.status.busy":"2025-12-12T17:46:57.959504Z","iopub.status.idle":"2025-12-12T17:46:57.984838Z","shell.execute_reply":"2025-12-12T17:46:57.983818Z","shell.execute_reply.started":"2025-12-12T17:46:57.959807Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == 'TRAINING':\n    train_dataframe = pl.read_csv(\"/kaggle/input/MABe-mouse-behavior-detection/train.csv\")\n    \n    train_dataframe = (\n        train_dataframe\n        .with_row_index()\n        .filter(pl.col(\"index\") != 8640)\n        .filter(pl.col(\"behaviors_labeled\").is_not_null())\n        .drop(\"index\")\n    )\n    \n    train_behavior_dataframe = (\n        train_dataframe.filter(pl.col(\"behaviors_labeled\").is_not_null())\n        .select(\n            pl.col(\"lab_id\"),\n            pl.col(\"video_id\"),\n            pl.col(\"behaviors_labeled\").map_elements(eval, return_dtype=pl.List(pl.Utf8)).alias(\"behaviors_labeled_list\"),\n        )\n        .explode(\"behaviors_labeled_list\")\n        .rename({\"behaviors_labeled_list\": \"behaviors_labeled_element\"})\n        .select(\n            pl.col(\"lab_id\"),\n            pl.col(\"video_id\"),\n            pl.col(\"behaviors_labeled_element\").str.split(\",\").list[0].str.replace_all(\"'\", \"\").alias(\"agent\"),\n            pl.col(\"behaviors_labeled_element\").str.split(\",\").list[1].str.replace_all(\"'\", \"\").alias(\"target\"),\n            pl.col(\"behaviors_labeled_element\").str.split(\",\").list[2].str.replace_all(\"'\", \"\").alias(\"behavior\"),\n        )\n    )\n    \n    train_self_behavior_dataframe = train_behavior_dataframe.filter(pl.col(\"behavior\").is_in(SELF_ACTIONS))\n    train_pair_behavior_dataframe = train_behavior_dataframe.filter(pl.col(\"behavior\").is_in(PAIR_ACTIONS))","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:57.986283Z","iopub.status.busy":"2025-12-12T17:46:57.985814Z","iopub.status.idle":"2025-12-12T17:46:58.079106Z","shell.execute_reply":"2025-12-12T17:46:58.078091Z","shell.execute_reply.started":"2025-12-12T17:46:57.986256Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_self_features(\n    metadata: dict,\n    tracking: pl.DataFrame,\n) -> pl.DataFrame:\n    def body_parts_distance(body_part_1, body_part_2):\n        assert body_part_1 in BODY_PARTS\n        assert body_part_2 in BODY_PARTS\n        return (\n            (pl.col(f\"agent_x_{body_part_1}\") - pl.col(f\"agent_x_{body_part_2}\")).pow(2)\n            + (pl.col(f\"agent_y_{body_part_1}\") - pl.col(f\"agent_y_{body_part_2}\")).pow(2)\n        ).sqrt() / metadata[\"pix_per_cm_approx\"]\n\n    def body_part_speed(body_part, period_ms):\n        assert body_part in BODY_PARTS\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        return (\n            ((pl.col(f\"agent_x_{body_part}\").diff()).pow(2) + (pl.col(f\"agent_y_{body_part}\").diff()).pow(2)).sqrt()\n            / metadata[\"pix_per_cm_approx\"]\n            * metadata[\"frames_per_second\"]\n        ).rolling_mean(window_size=window_frames, center=True, min_samples=1)\n    #Độ thuôn dài cơ thế\n    def elongation():\n        d1 = body_parts_distance(\"nose\", \"tail_base\")\n        d2 = body_parts_distance(\"ear_left\", \"ear_right\")\n        return d1 / (d2 + 1e-06)\n\n    def body_angle():\n        v1x = pl.col(\"agent_x_nose\") - pl.col(\"agent_x_body_center\")\n        v1y = pl.col(\"agent_y_nose\") - pl.col(\"agent_y_body_center\")\n        v2x = pl.col(\"agent_x_tail_base\") - pl.col(\"agent_x_body_center\")\n        v2y = pl.col(\"agent_y_tail_base\") - pl.col(\"agent_y_body_center\")\n        return (v1x * v2x + v1y * v2y) / ((v1x.pow(2) + v1y.pow(2)).sqrt() * (v2x.pow(2) + v2y.pow(2)).sqrt() + 1e-06)\n\n    def freeze_score(period_ms=1000):\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        speed = body_part_speed(\"body_center\", 200)\n        low_speed = (speed < 0.5).cast(pl.Float64)        \n        freeze_ratio = low_speed.rolling_mean(window_size=window_frames, center=True, min_samples=1)        \n        x_var = pl.col(\"agent_x_body_center\").rolling_var(window_size=window_frames, center=True, min_samples=1)\n        y_var = pl.col(\"agent_y_body_center\").rolling_var(window_size=window_frames, center=True, min_samples=1)\n        position_stability = 1.0 / (x_var + y_var + 1e-06)\n        return freeze_ratio * position_stability\n    \n    def movement_curvature(period_ms=500):\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        vel_x = pl.col(\"agent_x_body_center\").diff().fill_null(0)\n        vel_y = pl.col(\"agent_y_body_center\").diff().fill_null(0)\n        acc_x = vel_x.diff().fill_null(0)\n        acc_y = vel_y.diff().fill_null(0)\n        \n        cross_product = vel_x * acc_y - vel_y * acc_x\n        vel_magnitude = (vel_x.pow(2) + vel_y.pow(2)).sqrt() + 1e-06\n        \n        curvature = cross_product.abs() / vel_magnitude.pow(3)\n        return curvature.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n\n    def body_part_acceleration(body_part, period_ms):\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        speed = body_part_speed(body_part, period_ms)\n        return speed.diff().rolling_mean(window_size=window_frames, center=True, min_samples=1)\n\n    def body_part_jerk(body_part, period_ms):\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        accel = body_part_acceleration(body_part, period_ms)\n        return accel.diff().rolling_mean(window_size=window_frames, center=True, min_samples=1)\n\n    def head_lowering(period_ms=500):\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))        \n        head_height_diff = (pl.col(\"agent_y_body_center\") - pl.col(\"agent_y_nose\")) / metadata[\"pix_per_cm_approx\"]        \n        head_low_ratio = (head_height_diff > 2.0).cast(pl.Float64)\n        return head_low_ratio.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n\n    def grooming_features():\n        features = []        \n        for period_ms in [200, 500]:\n            window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n            nose_body_dist = body_parts_distance(\"nose\", \"body_center\")\n            dist_diff = nose_body_dist.diff().abs()\n            oscillation_freq = dist_diff.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n            features.append(oscillation_freq.alias(f\"agent__groom_oscillation_{period_ms}ms\"))\n        \n        area_window = max(1, int(round(2000 * metadata[\"frames_per_second\"] / 1000.0)))\n        x_range = pl.col(\"agent_x_body_center\").rolling_max(window_size=area_window) - \\\n                  pl.col(\"agent_x_body_center\").rolling_min(window_size=area_window)\n        y_range = pl.col(\"agent_y_body_center\").rolling_max(window_size=area_window) - \\\n                  pl.col(\"agent_y_body_center\").rolling_min(window_size=area_window)\n        activity_area = (x_range * y_range) / (metadata[\"pix_per_cm_approx\"] ** 2)\n        features.append(activity_area.alias(\"agent__grooming_activity_area\"))\n        \n        return features\n    \n    def head_orientation_change(period_ms=300):\n        \"\"\"\n        Tốc độ thay đổi hướng nhìn của đầu.\n        -> Hành vi: 'exploreobject' (lắc đầu qua lại ngửi đồ vật), 'sniff' (đánh hơi).\n        \"\"\"\n        v1x = pl.col(\"agent_x_nose\") - pl.col(\"agent_x_body_center\")\n        v1y = pl.col(\"agent_y_nose\") - pl.col(\"agent_y_body_center\")\n        angle = pl.arctan2(v1y, v1x)\n        \n        # Xử lý hiện tượng nhảy góc (Ví dụ từ 179 độ nhảy sang -179 độ là thay đổi nhỏ, không phải lớn)\n        angle_diff = angle.diff()\n        angle_diff = pl.when(angle_diff > 3.14159).then(angle_diff - 2*3.14159)\\\n                       .when(angle_diff < -3.14159).then(angle_diff + 2*3.14159)\\\n                       .otherwise(angle_diff)\n        \n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        return angle_diff.abs().rolling_mean(window_size=window_frames, center=True, min_samples=1)\n    \n    def body_compression():\n        \"\"\"\n        Độ nén của cơ thể (Tổng khoảng cách các chi đến mũi).\n        -> Hành vi: Nhỏ nhất khi 'huddle' (cuộn tròn), lớn nhất khi 'stretch'/'run'.\n        \"\"\"\n        return (\n            body_parts_distance(\"nose\", \"hip_left\")\n            + body_parts_distance(\"nose\", \"hip_right\")\n            + body_parts_distance(\"nose\", \"tail_base\")\n        )\n    \n    def tail_movement():\n        \"\"\"\n        Chuyển động riêng của đuôi.\n        -> Hành vi: Phân biệt 'rest' (đuôi im) với 'alert' (đuôi ve vẩy) hoặc giữ thăng bằng.\n        \"\"\"\n        return (\n            (pl.col(\"agent_x_tail_tip\").diff().pow(2)\n             + pl.col(\"agent_y_tail_tip\").diff().pow(2)).sqrt()\n            / metadata[\"pix_per_cm_approx\"]\n        )\n    \n    def alignment_score():\n        \"\"\"\n        Độ thẳng hàng: Mũi - Tâm - Đuôi có thẳng hàng không?\n        -> Hành vi: Thẳng (gần 1) -> 'run'. Cong (gần -1 hoặc 0) -> 'turn', 'groom'.\n        \"\"\"\n        nx = pl.col(\"agent_x_nose\") - pl.col(\"agent_x_body_center\")\n        ny = pl.col(\"agent_y_nose\") - pl.col(\"agent_y_body_center\")\n        tx = pl.col(\"agent_x_tail_base\") - pl.col(\"agent_x_body_center\")\n        ty = pl.col(\"agent_y_tail_base\") - pl.col(\"agent_y_body_center\")\n        return (nx * tx + ny * ty) / (\n            (nx.pow(2) + ny.pow(2)).sqrt() * (tx.pow(2) + ty.pow(2)).sqrt() + 1e-06\n        )\n    \n    def vibration_local():\n        \"\"\"\n        Độ rung tại chỗ (Tổng quãng đường di chuyển L1 cực nhỏ).\n        -> Hành vi: Phân biệt 'freeze' (đông cứng hoàn toàn) với 'shivering' (run rẩy) hoặc chuẩn bị nhảy.\n        \"\"\"\n        return (\n            pl.col(\"agent_x_body_center\").diff().abs()\n            + pl.col(\"agent_y_body_center\").diff().abs()\n        ).rolling_mean(window_size=5, center=True)\n\n    \n    n_mice = (\n        (metadata[\"mouse1_strain\"] is not None)\n        + (metadata[\"mouse2_strain\"] is not None)\n        + (metadata[\"mouse3_strain\"] is not None)\n        + (metadata[\"mouse4_strain\"] is not None)\n    )\n    \n    start_frame = tracking.select(pl.col(\"video_frame\").min()).item()\n    end_frame = tracking.select(pl.col(\"video_frame\").max()).item()\n\n    result = []\n\n    pivot = tracking.pivot(\n        on=[\"bodypart\"],\n        index=[\"video_frame\", \"mouse_id\"],\n        values=[\"x\", \"y\"],\n    ).sort([\"mouse_id\", \"video_frame\"])\n    \n    pivot_trackings = {mouse_id: pivot.filter(pl.col(\"mouse_id\") == mouse_id) for mouse_id in range(1, n_mice + 1)}\n\n    for agent_mouse_id in range(1, n_mice + 1):\n        result_element = pl.DataFrame(\n            {\n                \"video_id\": metadata[\"video_id\"],\n                \"agent_mouse_id\": agent_mouse_id,\n                \"target_mouse_id\": -1,\n                \"video_frame\": pl.arange(start_frame, end_frame + 1, eager=True),\n            },\n            schema={\n                \"video_id\": pl.Int32,\n                \"agent_mouse_id\": pl.Int8,\n                \"target_mouse_id\": pl.Int8,\n                \"video_frame\": pl.Int32,\n            },\n        )\n\n        pivot = pivot_trackings[agent_mouse_id].select(\n            pl.col(\"video_frame\"),\n            pl.exclude(\"video_frame\").name.prefix(\"agent_\"),\n        )\n        columns = pivot.columns\n        pivot = pivot.with_columns(\n            *[pl.lit(None).cast(pl.Float32).alias(f\"agent_x_{bp}\") for bp in BODY_PARTS if f\"agent_x_{bp}\" not in columns],\n            *[pl.lit(None).cast(pl.Float32).alias(f\"agent_y_{bp}\") for bp in BODY_PARTS if f\"agent_y_{bp}\" not in columns],\n        )\n\n        features = pivot.with_columns(\n            pl.lit(agent_mouse_id).alias(\"agent_mouse_id\"),\n            pl.lit(-1).alias(\"target_mouse_id\"),\n        ).select(\n            pl.col(\"video_frame\"),\n            pl.col(\"agent_mouse_id\"),\n            pl.col(\"target_mouse_id\"),\n            *[\n                body_parts_distance(body_part_1, body_part_2).alias(f\"aa__{body_part_1}__{body_part_2}__distance\")\n                for body_part_1, body_part_2 in itertools.combinations(BODY_PARTS, 2)\n            ],\n            *[\n                body_part_speed(body_part, period_ms).alias(f\"agent__{body_part}__speed_{period_ms}ms\")\n                for body_part, period_ms in itertools.product([\"ear_left\", \"ear_right\", \"tail_base\"], [500, 1000, 2000, 3000])\n            ],\n            elongation().alias(\"agent__elongation\"),\n            body_angle().alias(\"agent__body_angle\"),\n            freeze_score(1000).alias(\"agent__freeze_score\"),\n            movement_curvature(500).alias(\"agent__movement_curvature\"),\n            head_lowering(500).alias(\"agent__head_lowering\"),\n            body_part_acceleration(\"body_center\", 500).alias(\"agent__body_center__acceleration_500ms\"),\n            body_part_acceleration(\"nose\", 500).alias(\"agent__nose__acceleration_500ms\"),\n            body_part_jerk(\"body_center\", 500).alias(\"agent__body_center__jerk_500ms\"),\n            body_part_jerk(\"nose\", 500).alias(\"agent__nose__jerk_500ms\"),\n            *grooming_features(),\n            head_orientation_change(300).alias(\"agent__head_orientation_change_300ms\"),\n            body_compression().alias(\"agent__body_compression\"),\n            tail_movement().alias(\"agent__tail_movement\"),\n            alignment_score().alias(\"agent__alignment_score\"),\n            vibration_local().alias(\"agent__vibration_local\")\n        )\n\n        result_element = result_element.join(\n            features,\n            on=[\"video_frame\", \"agent_mouse_id\", \"target_mouse_id\"],\n            how=\"left\",\n        )\n        result.append(result_element)\n\n    return pl.concat(result, how=\"vertical\")","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:58.136279Z","iopub.status.busy":"2025-12-12T17:46:58.135974Z","iopub.status.idle":"2025-12-12T17:46:58.178082Z","shell.execute_reply":"2025-12-12T17:46:58.177129Z","shell.execute_reply.started":"2025-12-12T17:46:58.136258Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_pair_features(\n    metadata: dict,\n    tracking: pl.DataFrame,\n) -> pl.DataFrame:\n    \n    def body_parts_distance(agent_or_target_1, body_part_1, agent_or_target_2, body_part_2):\n        assert agent_or_target_1 == \"agent\" or agent_or_target_1 == \"target\"\n        assert agent_or_target_2 == \"agent\" or agent_or_target_2 == \"target\"\n        assert body_part_1 in BODY_PARTS\n        assert body_part_2 in BODY_PARTS\n        return (\n            (pl.col(f\"{agent_or_target_1}_x_{body_part_1}\") - pl.col(f\"{agent_or_target_2}_x_{body_part_2}\")).pow(2)\n            + (pl.col(f\"{agent_or_target_1}_y_{body_part_1}\") - pl.col(f\"{agent_or_target_2}_y_{body_part_2}\")).pow(2)\n        ).sqrt() / metadata[\"pix_per_cm_approx\"]\n\n    def body_part_speed(agent_or_target, body_part, period_ms):\n        assert agent_or_target == \"agent\" or agent_or_target == \"target\"\n        assert body_part in BODY_PARTS\n        window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n        return (\n            (\n                (pl.col(f\"{agent_or_target}_x_{body_part}\").diff()).pow(2)\n                + (pl.col(f\"{agent_or_target}_y_{body_part}\").diff()).pow(2)\n            ).sqrt()\n            / metadata[\"pix_per_cm_approx\"]\n            * metadata[\"frames_per_second\"]\n        ).rolling_mean(window_size=window_frames, center=True)\n\n    def elongation(agent_or_target):\n        assert agent_or_target == \"agent\" or agent_or_target == \"target\"\n        d1 = body_parts_distance(agent_or_target, \"nose\", agent_or_target, \"tail_base\")\n        d2 = body_parts_distance(agent_or_target, \"ear_left\", agent_or_target, \"ear_right\")\n        return d1 / (d2 + 1e-06)\n\n    def body_angle(agent_or_target):\n        assert agent_or_target == \"agent\" or agent_or_target == \"target\"\n        v1x = pl.col(f\"{agent_or_target}_x_nose\") - pl.col(f\"{agent_or_target}_x_body_center\")\n        v1y = pl.col(f\"{agent_or_target}_y_nose\") - pl.col(f\"{agent_or_target}_y_body_center\")\n        v2x = pl.col(f\"{agent_or_target}_x_tail_base\") - pl.col(f\"{agent_or_target}_x_body_center\")\n        v2y = pl.col(f\"{agent_or_target}_y_tail_base\") - pl.col(f\"{agent_or_target}_y_body_center\")\n        return (v1x * v2x + v1y * v2y) / ((v1x.pow(2) + v1y.pow(2)).sqrt() * (v2x.pow(2) + v2y.pow(2)).sqrt() + 1e-06)\n\n    def approach_features():\n        current_dist = body_parts_distance(\"agent\", \"body_center\", \"target\", \"body_center\")\n        features = []\n        for lag_ms in [100, 250, 500]:\n            lag_frames = max(1, int(round(lag_ms * metadata[\"frames_per_second\"] / 1000.0)))\n            \n            past_dist = current_dist.shift(lag_frames).fill_null(strategy=\"forward\")\n            approach_rate = (past_dist - current_dist) / (lag_ms / 1000.0)  # cm/s\n            \n            for period_ms in [500, 1000]:\n                window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n                mean_rate = approach_rate.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n                features.append(mean_rate.alias(f\"approach_rate_{lag_ms}lag_{period_ms}avg\"))\n        \n        return features\n\n    def relative_orientation_features():\n        dx = pl.col(\"target_x_body_center\") - pl.col(\"agent_x_body_center\")\n        dy = pl.col(\"target_y_body_center\") - pl.col(\"agent_y_body_center\")        \n        agent_dir_x = pl.col(\"agent_x_nose\") - pl.col(\"agent_x_tail_base\")\n        agent_dir_y = pl.col(\"agent_y_nose\") - pl.col(\"agent_y_tail_base\")        \n        dot = dx * agent_dir_x + dy * agent_dir_y\n        mag_agent = (agent_dir_x.pow(2) + agent_dir_y.pow(2)).sqrt() + 1e-06\n        mag_to_target = (dx.pow(2) + dy.pow(2)).sqrt() + 1e-06\n        \n        facing_score = dot / (mag_agent * mag_to_target)\n        \n        features = []\n        \n        facing_each_other = (facing_score > 0.7).cast(pl.Float64)\n        side_by_side = ((facing_score >= -0.3) & (facing_score <= 0.3)).cast(pl.Float64)\n        facing_away = (facing_score < -0.7).cast(pl.Float64)\n        \n        features.extend([\n            facing_each_other.alias(\"facing_each_other\"),\n            side_by_side.alias(\"side_by_side\"),\n            facing_away.alias(\"facing_away\"),\n            facing_score.alias(\"relative_orientation_score\"),\n        ])\n        \n        return features\n\n    def chase_features():\n        dist = body_parts_distance(\"agent\", \"body_center\", \"target\", \"body_center\")        \n        agent_vx = pl.col(\"agent_x_body_center\").diff()\n        agent_vy = pl.col(\"agent_y_body_center\").diff()\n        target_vx = pl.col(\"target_x_body_center\").diff()\n        target_vy = pl.col(\"target_y_body_center\").diff()\n        agent_speed = (agent_vx.pow(2) + agent_vy.pow(2)).sqrt() * metadata[\"frames_per_second\"]\n        target_speed = (target_vx.pow(2) + target_vy.pow(2)).sqrt() * metadata[\"frames_per_second\"]        \n        dx = pl.col(\"target_x_body_center\") - pl.col(\"agent_x_body_center\")\n        dy = pl.col(\"target_y_body_center\") - pl.col(\"agent_y_body_center\")        \n        agent_proj = (agent_vx * dx + agent_vy * dy) / (dist + 1e-06)\n        target_proj = (target_vx * (-dx) + target_vy * (-dy)) / (dist + 1e-06)\n        features = []\n        chase_score = (agent_proj > 0).cast(pl.Float64) * (target_proj > 0).cast(pl.Float64)\n        speed_diff = agent_speed - target_speed\n        for period_ms in [500, 1000, 2000]:\n            window_frames = max(1, int(round(period_ms * metadata[\"frames_per_second\"] / 1000.0)))\n            features.extend([\n                chase_score.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n                .alias(f\"chase_score_{period_ms}ms\"),\n                speed_diff.rolling_mean(window_size=window_frames, center=True, min_samples=1)\n                .alias(f\"speed_difference_{period_ms}ms\"),\n            ])\n        return features\n    \n    def mounting_features():\n        height_diff = (pl.col(\"target_y_body_center\") - pl.col(\"agent_y_body_center\")) / metadata[\"pix_per_cm_approx\"]        \n        vertical_dist = height_diff.abs()        \n        agent_below_target = (height_diff > 5.0).cast(pl.Float64)        \n        dx = pl.col(\"target_x_body_center\") - pl.col(\"agent_x_body_center\")\n        dy = pl.col(\"target_y_body_center\") - pl.col(\"agent_y_body_center\")        \n        target_dir_x = pl.col(\"target_x_nose\") - pl.col(\"target_x_tail_base\")\n        target_dir_y = pl.col(\"target_y_nose\") - pl.col(\"target_y_tail_base\")        \n        behind_score = -(dx * target_dir_x + dy * target_dir_y) / ((dx.pow(2) + dy.pow(2)).sqrt() * (target_dir_x.pow(2) + target_dir_y.pow(2)).sqrt() + 1e-06)\n        features = [\n            height_diff.alias(\"mounting_height_difference\"),\n            vertical_dist.alias(\"mounting_vertical_distance\"),\n            agent_below_target.alias(\"agent_below_target\"),\n            behind_score.alias(\"mounting_behind_score\"),\n        ]\n        \n        return features\n\n\n    def social_contact_features():\n        # Chỉ tính toán để lấy logic tiếp xúc (Duration), không output lại cột distance thừa\n        \n        # Tận dụng các cột đã tính ở bước trước (cần đảm bảo tên cột khớp logic)\n        # Hoặc tính lại biến tạm (như dưới đây) nhưng KHÔNG append vào list kết quả\n        d_nose_nose = body_parts_distance(\"agent\", \"nose\", \"target\", \"nose\")\n        d_nose_tail = body_parts_distance(\"agent\", \"nose\", \"target\", \"tail_base\")\n        d_nose_body = body_parts_distance(\"agent\", \"nose\", \"target\", \"body_center\")\n\n        # Chỉ trả về các Feature \"Thời lượng\" (Duration) - Đây là cái mới\n        return [\n            # Tỉ lệ tiếp xúc trong 1 giây (30 frames)\n            (d_nose_tail < 2.0).cast(pl.Float64).rolling_sum(window_size=30, center=True).alias(\"contact_genital_duration_1s\"),\n            (d_nose_nose < 2.0).cast(pl.Float64).rolling_sum(window_size=30, center=True).alias(\"contact_face_duration_1s\"),\n            (d_nose_body < 2.0).cast(pl.Float64).rolling_sum(window_size=30, center=True).alias(\"contact_body_duration_1s\"),\n        ]\n    \n    n_mice = (\n        (metadata[\"mouse1_strain\"] is not None)\n        + (metadata[\"mouse2_strain\"] is not None)\n        + (metadata[\"mouse3_strain\"] is not None)\n        + (metadata[\"mouse4_strain\"] is not None)\n    )\n    \n    start_frame = tracking.select(pl.col(\"video_frame\").min()).item()\n    end_frame = tracking.select(pl.col(\"video_frame\").max()).item()\n\n    result = []\n\n    pivot = tracking.pivot(\n        on=[\"bodypart\"],\n        index=[\"video_frame\", \"mouse_id\"],\n        values=[\"x\", \"y\"],\n    ).sort([\"mouse_id\", \"video_frame\"])\n    \n    pivot_trackings = {mouse_id: pivot.filter(pl.col(\"mouse_id\") == mouse_id) for mouse_id in range(1, n_mice + 1)}\n\n    for agent_mouse_id, target_mouse_id in itertools.permutations(range(1, n_mice + 1), 2):\n        result_element = pl.DataFrame(\n            {\n                \"video_id\": metadata[\"video_id\"],\n                \"agent_mouse_id\": agent_mouse_id,\n                \"target_mouse_id\": target_mouse_id,\n                \"video_frame\": pl.arange(start_frame, end_frame + 1, eager=True),\n            },\n            schema={\n                \"video_id\": pl.Int32,\n                \"agent_mouse_id\": pl.Int8,\n                \"target_mouse_id\": pl.Int8,\n                \"video_frame\": pl.Int32,\n            },\n        )\n\n        merged_pivot = (\n            pivot_trackings[agent_mouse_id]\n            .select(\n                pl.col(\"video_frame\"),\n                pl.exclude(\"video_frame\").name.prefix(\"agent_\"),\n            )\n            .join(\n                pivot_trackings[target_mouse_id].select(\n                    pl.col(\"video_frame\"),\n                    pl.exclude(\"video_frame\").name.prefix(\"target_\"),\n                ),\n                on=\"video_frame\",\n                how=\"inner\",\n            )\n        )\n        \n        columns = merged_pivot.columns\n        merged_pivot = merged_pivot.with_columns(\n            *[pl.lit(None).cast(pl.Float32).alias(f\"agent_x_{bp}\") for bp in BODY_PARTS if f\"agent_x_{bp}\" not in columns],\n            *[pl.lit(None).cast(pl.Float32).alias(f\"agent_y_{bp}\") for bp in BODY_PARTS if f\"agent_y_{bp}\" not in columns],\n            *[pl.lit(None).cast(pl.Float32).alias(f\"target_x_{bp}\") for bp in BODY_PARTS if f\"target_x_{bp}\" not in columns],\n            *[pl.lit(None).cast(pl.Float32).alias(f\"target_y_{bp}\") for bp in BODY_PARTS if f\"target_y_{bp}\" not in columns],\n        )\n\n        features = merged_pivot.with_columns(\n            pl.lit(agent_mouse_id).alias(\"agent_mouse_id\"),\n            pl.lit(target_mouse_id).alias(\"target_mouse_id\"),\n        ).select(\n            pl.col(\"video_frame\"),\n            pl.col(\"agent_mouse_id\"),\n            pl.col(\"target_mouse_id\"),\n            *[\n                body_parts_distance(\"agent\", agent_body_part, \"target\", target_body_part).alias(\n                    f\"at__{agent_body_part}__{target_body_part}__distance\"\n                )\n                for agent_body_part, target_body_part in itertools.product(BODY_PARTS, repeat=2)\n            ],\n            *[\n                body_part_speed(\"agent\", body_part, period_ms).alias(f\"agent__{body_part}__speed_{period_ms}ms\")\n                for body_part, period_ms in itertools.product([\"ear_left\", \"ear_right\", \"tail_base\"], [500, 1000, 2000, 3000])\n            ],\n            *[\n                body_part_speed(\"target\", body_part, period_ms).alias(f\"target__{body_part}__speed_{period_ms}ms\")\n                for body_part, period_ms in itertools.product([\"ear_left\", \"ear_right\", \"tail_base\"], [500, 1000, 2000, 3000])\n            ],\n            elongation(\"agent\").alias(\"agent__elongation\"),\n            elongation(\"target\").alias(\"target__elongation\"),\n            body_angle(\"agent\").alias(\"agent__body_angle\"),\n            body_angle(\"target\").alias(\"target__body_angle\"),\n\n            *approach_features(),\n            *relative_orientation_features(),\n            *chase_features(),\n            *mounting_features(),\n            *social_contact_features()\n        )\n\n        result_element = result_element.join(\n            features,\n            on=[\"video_frame\", \"agent_mouse_id\", \"target_mouse_id\"],\n            how=\"left\",\n        )\n        result.append(result_element)\n\n    return pl.concat(result, how=\"vertical\")","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:58.180846Z","iopub.status.busy":"2025-12-12T17:46:58.180497Z","iopub.status.idle":"2025-12-12T17:46:58.216101Z","shell.execute_reply":"2025-12-12T17:46:58.214991Z","shell.execute_reply.started":"2025-12-12T17:46:58.180827Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_video(row):\n    lab_id = row[\"lab_id\"]\n    video_id = row[\"video_id\"]\n\n    tracking_path = f\"/kaggle/input/MABe-mouse-behavior-detection/train_tracking/{lab_id}/{video_id}.parquet\"\n    tracking = pl.read_parquet(tracking_path)\n\n    self_features = make_self_features(metadata=row, tracking=tracking)\n    pair_features = make_pair_features(metadata=row, tracking=tracking)\n\n    self_features.write_parquet(f\"self_features/{video_id}.parquet\")\n    pair_features.write_parquet(f\"pair_features/{video_id}.parquet\")\n\n    return video_id\n\nif STATUS == \"TRAINING\":\n    os.makedirs(\"self_features\", exist_ok=True)\n    os.makedirs(\"pair_features\", exist_ok=True)\n    rows = list(train_dataframe.filter(pl.col(\"behaviors_labeled\").is_not_null()).rows(named=True))\n    results = joblib.Parallel(n_jobs=-1, verbose=5)(joblib.delayed(process_video)(row) for row in rows)\n    del rows, results\n    gc.collect()","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:46:58.217723Z","iopub.status.busy":"2025-12-12T17:46:58.217405Z","iopub.status.idle":"2025-12-12T17:52:49.643149Z","shell.execute_reply":"2025-12-12T17:52:49.642342Z","shell.execute_reply.started":"2025-12-12T17:46:58.217690Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\ndef score_for_threshold(th, oof, y):\n    preds = (oof >= th)\n    return f1_score(y, preds, zero_division=0)\n\ndef tune_threshold(oof, y, n_jobs=-1):\n    thresholds = np.arange(0.0, 0.91, 0.01)\n    f1_scores = Parallel(n_jobs=n_jobs)(\n        delayed(score_for_threshold)(th, oof, y) for th in thresholds\n    )\n    f1_scores = np.array(f1_scores)\n\n    best_index = np.argmax(f1_scores)\n    best_threshold = thresholds[best_index]\n    best_f1 = f1_scores[best_index]\n\n    return best_threshold, best_f1\n\n\ndef tune_model(X, y, groups, type_model):\n    gkf = GroupKFold(n_splits=3)\n    oof_predictions = np.zeros(len(y))\n    best_iterations = []\n    for fold, (train_idx, val_idx) in enumerate(gkf.split(X, y, groups), 1):\n        X_train, X_val = X[train_idx], X[val_idx]\n        y_train, y_val = y[train_idx], y[val_idx]\n\n        pos_count = y_train.sum()\n        neg_count = len(y_train) - pos_count\n        pos_weight = neg_count / max(pos_count, 1)\n        pos_weight = min(pos_weight, 50)\n        \n        if type_model == 'x':\n            xgb_config = MODEL_CONFIG['x'].copy()\n            xgb_config['scale_pos_weight'] = pos_weight\n            model = xgb.XGBClassifier(**xgb_config)\n            \n        elif type_model == 'c':\n            cat_config = MODEL_CONFIG['c'].copy()\n            cat_config['scale_pos_weight'] = pos_weight\n            model = CatBoostClassifier(**cat_config)\n            \n        else:\n            lgb_config = MODEL_CONFIG['l'].copy()\n            lgb_config['scale_pos_weight'] = pos_weight\n            model = lgb.LGBMClassifier(**lgb_config)\n\n        model.fit(\n                X_train, y_train,\n                eval_set=[(X_val, y_val)],\n                eval_metric='binary_logloss',\n                callbacks=[\n                    lgb.early_stopping(stopping_rounds=50, verbose=False),\n                    lgb.log_evaluation(period=0) # Tắt log cho gọn\n                ]\n            )\n        # Lưu lại số cây tốt nhất để dùng cho Final Model\n        best_iterations.append(model.best_iteration_)\n        val_probs = model.predict_proba(X_val)[:, 1]\n        oof_predictions[val_idx] = val_probs\n\n        del model, X_train, X_val, y_train, y_val\n        gc.collect()\n        torch.cuda.empty_cache()\n\n    best_threshold, best_f1 = tune_threshold(oof_predictions, y)\n    final_n_estimators = int(np.mean(best_iterations)) if best_iterations else 300\n    total_pos_count = y.sum()\n    total_neg_count = len(y) - total_pos_count\n    total_pos_weight = total_neg_count / max(total_pos_count, 1)\n    \n    if type_model == 'x':\n        xgb_config = MODEL_CONFIG['x'].copy()\n        xgb_config['scale_pos_weight'] = total_pos_weight\n        final_model = xgb.XGBClassifier(**xgb_config)\n        \n    elif type_model == 'c':\n        cat_config = MODEL_CONFIG['c'].copy()\n        cat_config['scale_pos_weight'] = total_pos_weight\n        final_model = CatBoostClassifier(**cat_config)\n        \n    else:\n        lgb_config = MODEL_CONFIG['l'].copy()\n        lgb_config['scale_pos_weight'] = total_pos_weight\n        lgb_config.update({\n            'n_estimators': final_n_estimators,\n        })\n        final_model = lgb.LGBMClassifier(**lgb_config)\n\n    final_model.fit(X, y)\n\n    return final_model, best_threshold, best_f1","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:52:49.645135Z","iopub.status.busy":"2025-12-12T17:52:49.644890Z","iopub.status.idle":"2025-12-12T17:52:49.656779Z","shell.execute_reply":"2025-12-12T17:52:49.655755Z","shell.execute_reply.started":"2025-12-12T17:52:49.645115Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_NEGATIVES = 3500000\n\ndef train_validate_1(lab_id: str, behavior: str,\n                     indices: pl.DataFrame, features: pl.DataFrame, labels: pl.Series,\n                     max_negatives: int = MAX_NEGATIVES):\n    \n    result_dir = f\"results/{lab_id}/{behavior}\"\n    os.makedirs(result_dir, exist_ok=True)\n    \n    if labels.sum() == 0:\n        return None, 0.0\n    \n    # Nối features và labels thành DataFrame tạm\n    df = features.with_columns([\n        labels.alias(\"label\"),\n        indices[\"video_id\"].alias(\"video_id\") \n    ])\n    \n    # Tách nhãn 1 và 0\n    pos_df = df.filter(pl.col(\"label\") == 1)\n    neg_df = df.filter(pl.col(\"label\") == 0)\n    \n    # Giới hạn số nhãn 0\n    if len(neg_df) > max_negatives:\n        neg_df = neg_df.sample(n=max_negatives, shuffle=True)\n    \n    \n    df_sampled = pl.concat([pos_df, neg_df])\n    groups = df_sampled[\"video_id\"].to_numpy()\n\n    X = df_sampled.drop([\"label\", \"video_id\"]).to_numpy()\n    y = df_sampled[\"label\"].to_numpy()\n    \n    final_lgb, th_lgb, f1_lgb = tune_model(X, y, groups, \"l\")\n    \n    save_dir = os.path.join(result_dir, \"models_1\")\n    os.makedirs(save_dir, exist_ok=True)\n    \n    if final_lgb:\n        final_lgb.booster_.save_model(os.path.join(save_dir, \"model_l.txt\"))\n        with open(os.path.join(save_dir, \"threshold_l.txt\"), \"w\") as f:\n            f.write(str(th_lgb))\n        print(f'LGB_Level1: th-{th_lgb:.3f}, f1-{f1_lgb:.3f}')\n        return final_lgb, th_lgb\n    \n    return None, 0.0\n\ndef train_validate_2(lab_id: str, behavior: str,\n                     indices: pl.DataFrame, features: pl.DataFrame, labels: pl.Series,\n                     model1_dict: dict,\n                     max_negatives: int = MAX_NEGATIVES):\n    \n    result_dir = f\"results/{lab_id}/{behavior}\"\n    os.makedirs(result_dir, exist_ok=True)\n    \n    if labels.sum() == 0:\n        return None, 0.0\n    \n    # Nối features và labels thành DataFrame tạm\n    df = features.with_columns([\n        labels.alias(\"label\"),\n        indices.select(pl.col(\"video_id\"))\n    ])\n    \n    # Tách nhãn 1 và 0\n    pos_df = df.filter(pl.col(\"label\") == 1)\n    neg_df = df.filter(pl.col(\"label\") == 0)\n    \n    # Giới hạn số nhãn 0\n    if len(neg_df) > max_negatives:\n        neg_df = neg_df.sample(n=max_negatives, shuffle=True)\n    \n    # Nối lại\n    df_sampled = pl.concat([pos_df, neg_df])\n    df_sampled = df_sampled.sample(n=df_sampled.height, shuffle=True)\n    groups = df_sampled[\"video_id\"].to_numpy()\n\n    X_original = df_sampled.drop([\"label\", \"video_id\"]).to_numpy()\n    y = df_sampled[\"label\"].to_numpy()\n    \n    predictions_list = []\n\n    for other_lab_id, (model1, thr) in model1_dict.get(behavior, {}).items():  \n        if other_lab_id == lab_id:\n            continue\n        try:\n            prob = model1.predict_proba(X_original)[:, 1]\n            pred_binary = (prob >= thr).astype(float)\n            predictions_list.append(prob.reshape(-1, 1))\n            predictions_list.append(pred_binary.reshape(-1, 1))\n        except Exception:\n            continue\n\n    if predictions_list:\n        predictions_array = np.hstack(predictions_list)\n        X_enhanced = np.hstack([X_original, predictions_array])\n    else:\n        X_enhanced = X_original\n    \n    final_lgb, th_lgb, f1_lgb = tune_model(X_enhanced, y, groups, \"l\")\n    \n    save_dir = os.path.join(result_dir, \"models_2\")\n    os.makedirs(save_dir, exist_ok=True)\n    \n    if final_lgb:\n        final_lgb.booster_.save_model(os.path.join(save_dir, \"model_l.txt\"))\n        with open(os.path.join(save_dir, \"threshold_l.txt\"), \"w\") as f:\n            f.write(str(th_lgb))\n        print(f'LGB_Level2: th-{th_lgb:.3f}, f1-{f1_lgb:.3f}')\n        return final_lgb, th_lgb\n        \n    return None, 0.0\n","metadata":{"execution":{"iopub.execute_input":"2025-12-12T17:52:49.657885Z","iopub.status.busy":"2025-12-12T17:52:49.657660Z","iopub.status.idle":"2025-12-12T17:52:49.683366Z","shell.execute_reply":"2025-12-12T17:52:49.682411Z","shell.execute_reply.started":"2025-12-12T17:52:49.657866Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == 'TRAINING':\n    print(\"=\" * 60)\n    print(\"LEVEL 1\")\n    print(\"=\" * 60)\n    \n    groups = train_self_behavior_dataframe.group_by([\"lab_id\", \"behavior\"], maintain_order=True)\n    groups_list = list(groups)\n    total = len(groups_list)\n    \n    model1_dict = {}\n    \n    for (lab_id, behavior), group in tqdm(groups_list, total=total):\n        print(f\"\\n=== LEVEL1 - LAB: {lab_id} | BEHAVIOR: {behavior} ===\")\n        \n        index_list = []\n        feature_list = []\n        label_list = []\n        \n        for row in group.iter_rows(named=True):\n            video_id = row[\"video_id\"]\n            agent = row[\"agent\"]\n            agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n            \n            feature_path = f\"self_features/{video_id}.parquet\"\n            try:\n                data = pl.scan_parquet(feature_path).filter(\n                    pl.col(\"agent_mouse_id\") == agent_mouse_id\n                )\n            except Exception as e:\n                continue\n            \n            index = data.select(INDEX_COLS).collect(engine=\"streaming\")\n            feature = data.select(pl.exclude(INDEX_COLS)).collect(engine=\"streaming\")\n            \n            ann_path = f\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/{lab_id}/{video_id}.parquet\"\n            if os.path.exists(ann_path):\n                annotation = (\n                    pl.scan_parquet(ann_path)\n                    .filter(\n                        (pl.col(\"action\") == behavior) &\n                        (pl.col(\"agent_id\") == agent_mouse_id)\n                    )\n                    .collect()\n                )\n            else:\n                annotation = pl.DataFrame()\n            \n            label_frames = set()\n            for ann in annotation.iter_rows(named=True):\n                label_frames.update(range(ann[\"start_frame\"], ann[\"stop_frame\"]))\n            \n            label = index.select(\n                pl.col(\"video_frame\").is_in(label_frames).cast(pl.Int8).alias(\"label\")\n            )\n            \n            if label[\"label\"].sum() == 0:\n                continue\n            \n            index_list.append(index)\n            feature_list.append(feature)\n            label_list.append(label[\"label\"])\n        \n        if not index_list:\n            continue\n        \n        indices = pl.concat(index_list)\n        features = pl.concat(feature_list)\n        labels = pl.concat(label_list)\n\n        if labels.min() == labels.max():\n            print(f\"Skipping {behavior} in {lab_id}: All labels are {labels.max()}\")\n            continue\n        \n        print(f\"Samples: {len(indices)}\")\n        print(f\"Positive: {labels.sum()}\")\n        print(f\"Features: {len(features.columns)}\")\n        \n        final_lgb, thr_lgb = train_validate_1(lab_id, behavior, indices, features, labels)\n        \n        if final_lgb is not None:\n            if behavior not in model1_dict:\n                model1_dict[behavior] = {}\n            model1_dict[behavior][lab_id] = (final_lgb, thr_lgb)\n        \n        del indices, features, labels, index_list, feature_list, label_list\n        gc.collect()\n    \n    # print(\"\\n\" + \"=\" * 60)\n    # print(\"LEVEL 2\")\n    # print(\"=\" * 60)\n    \n    # for (lab_id, behavior), group in tqdm(groups_list, total=total):\n    #     print(f\"\\n=== LEVEL2 - LAB: {lab_id} | BEHAVIOR: {behavior} ===\")\n        \n    #     if behavior not in model1_dict:\n    #         continue\n        \n    #     index_list = []\n    #     feature_list = []\n    #     label_list = []\n        \n    #     for row in group.iter_rows(named=True):\n    #         video_id = row[\"video_id\"]\n    #         agent = row[\"agent\"]\n    #         agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n            \n    #         feature_path = f\"self_features/{video_id}.parquet\"\n    #         data = pl.scan_parquet(feature_path).filter(\n    #             pl.col(\"agent_mouse_id\") == agent_mouse_id\n    #         )\n            \n    #         index = data.select(INDEX_COLS).collect(engine=\"streaming\")\n    #         feature = data.select(pl.exclude(INDEX_COLS)).collect(engine=\"streaming\")\n            \n    #         ann_path = f\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/{lab_id}/{video_id}.parquet\"\n    #         if os.path.exists(ann_path):\n    #             annotation = (\n    #                 pl.scan_parquet(ann_path)\n    #                 .filter(\n    #                     (pl.col(\"action\") == behavior) &\n    #                     (pl.col(\"agent_id\") == agent_mouse_id)\n    #                 )\n    #                 .collect()\n    #             )\n    #         else:\n    #             annotation = pl.DataFrame()\n            \n    #         label_frames = set()\n    #         for ann in annotation.iter_rows(named=True):\n    #             label_frames.update(range(ann[\"start_frame\"], ann[\"stop_frame\"]))\n            \n    #         label = index.select(\n    #             pl.col(\"video_frame\").is_in(label_frames).cast(pl.Int8).alias(\"label\")\n    #         )\n            \n    #         if label[\"label\"].sum() == 0:\n    #             continue\n            \n    #         index_list.append(index)\n    #         feature_list.append(feature)\n    #         label_list.append(label[\"label\"])\n        \n    #     if not index_list:\n    #         continue\n        \n    #     indices = pl.concat(index_list)\n    #     features = pl.concat(feature_list)\n    #     labels = pl.concat(label_list)\n\n    #     if labels.min() == labels.max():\n    #         continue\n        \n    #     print(f\"Samples: {len(indices)}\")\n    #     print(f\"Positive: {labels.sum()}\")\n    #     print(f\"Features: {len(features.columns)}\")\n        \n    #     train_validate_2(lab_id, behavior, indices, features, labels, model1_dict)\n        \n    #     del indices, features, labels\n    #     gc.collect()\n    del model1_dict\n    gc.collect()","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.256Z","iopub.execute_input":"2025-12-12T17:52:49.684653Z","iopub.status.busy":"2025-12-12T17:52:49.684348Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == 'TRAINING':\n    print(\"=\" * 60)\n    print(\"PAIR BEHAVIORS - LEVEL 1\")\n    print(\"=\" * 60)\n    \n    groups = train_pair_behavior_dataframe.group_by(\n        [\"lab_id\", \"behavior\"], \n        maintain_order=True\n    )\n    groups_list = list(groups)\n    total = len(groups_list)\n    \n    model1_dict = {}\n    \n    for (lab_id, behavior), group in tqdm(groups_list, total=total):\n        print(f\"\\n=== LEVEL1_PAIR - LAB: {lab_id} | BEHAVIOR: {behavior} ===\")\n\n        index_list = []\n        feature_list = []\n        label_list = []\n\n        for row in group.iter_rows(named=True):\n            video_id = row[\"video_id\"]\n            agent = row[\"agent\"]\n            target = row[\"target\"]\n\n            agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n            target_mouse_id = int(re.search(r\"mouse(\\d+)\", target).group(1))\n\n            feature_path = f\"pair_features/{video_id}.parquet\"\n            if not os.path.exists(feature_path):\n                continue\n\n            try:\n                data = (\n                    pl.scan_parquet(feature_path)\n                    .filter(\n                        (pl.col(\"agent_mouse_id\") == agent_mouse_id) &\n                        (pl.col(\"target_mouse_id\") == target_mouse_id)\n                    )\n                )\n                index = data.select(INDEX_COLS).collect(engine=\"streaming\")\n                feature = data.select(pl.exclude(INDEX_COLS)).collect(engine=\"streaming\")\n            except Exception as e:\n                # print(f\"Error reading features for {video_id}: {e}\")\n                continue\n\n            ann_path = (\n                f\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/\"\n                f\"{lab_id}/{video_id}.parquet\"\n            )\n\n            if os.path.exists(ann_path):\n                annotation = (\n                    pl.scan_parquet(ann_path)\n                    .filter(\n                        (pl.col(\"action\") == behavior) &\n                        (pl.col(\"agent_id\") == agent_mouse_id) &\n                        (pl.col(\"target_id\") == target_mouse_id)\n                    )\n                    .collect()\n                )\n            else:\n                annotation = pl.DataFrame()\n\n            label_frames = set()\n            for ann in annotation.iter_rows(named=True):\n                label_frames.update(range(ann[\"start_frame\"], ann[\"stop_frame\"]))\n\n            label = index.select(\n                pl.col(\"video_frame\")\n                .is_in(label_frames)\n                .cast(pl.Int8)\n                .alias(\"label\")\n            )\n\n            if label[\"label\"].sum() == 0:\n                continue\n\n            index_list.append(index)\n            feature_list.append(feature)\n            label_list.append(label[\"label\"])\n\n        if not index_list:\n            continue\n\n        indices = pl.concat(index_list)\n        features = pl.concat(feature_list)\n        labels = pl.concat(label_list)\n        \n        if labels.min() == labels.max():\n            print(f\"Skipping {behavior}: All labels are {labels.max()}\")\n            continue\n\n        print(f\"Samples: {len(indices)}\")\n        print(f\"Positive: {labels.sum()}\")\n        print(f\"Features: {len(features.columns)}\")\n\n        final_lgb, thr_lgb = train_validate_1(lab_id, behavior, indices, features, labels)\n        \n        if final_lgb is not None:\n            if behavior not in model1_dict:\n                model1_dict[behavior] = {}\n            model1_dict[behavior][lab_id] = (final_lgb, thr_lgb)\n        \n        del indices, features, labels, index_list, feature_list, label_list\n        gc.collect()\n    \n    # print(\"\\n\" + \"=\" * 60)\n    # print(\"PAIR BEHAVIORS - LEVEL 2\")\n    # print(\"=\" * 60)\n    \n    # for (lab_id, behavior), group in tqdm(groups_list, total=total):\n    #     print(f\"\\n=== LEVEL2_PAIR - LAB: {lab_id} | BEHAVIOR: {behavior} ===\")\n        \n    #     if behavior not in model1_dict:\n    #         continue\n        \n    #     index_list = []\n    #     feature_list = []\n    #     label_list = []\n\n    #     for row in group.iter_rows(named=True):\n    #         video_id = row[\"video_id\"]\n    #         agent = row[\"agent\"]\n    #         target = row[\"target\"]\n\n    #         agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n    #         target_mouse_id = int(re.search(r\"mouse(\\d+)\", target).group(1))\n\n    #         feature_path = f\"pair_features/{video_id}.parquet\"\n    #         if not os.path.exists(feature_path):\n    #             continue\n\n    #         data = (\n    #             pl.scan_parquet(feature_path)\n    #             .filter(\n    #                 (pl.col(\"agent_mouse_id\") == agent_mouse_id) &\n    #                 (pl.col(\"target_mouse_id\") == target_mouse_id)\n    #             )\n    #         )\n\n    #         index = data.select(INDEX_COLS).collect(engine=\"streaming\")\n    #         feature = data.select(pl.exclude(INDEX_COLS)).collect(engine=\"streaming\")\n\n    #         ann_path = (\n    #             f\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/\"\n    #             f\"{lab_id}/{video_id}.parquet\"\n    #         )\n\n    #         if os.path.exists(ann_path):\n    #             annotation = (\n    #                 pl.scan_parquet(ann_path)\n    #                 .filter(\n    #                     (pl.col(\"action\") == behavior) &\n    #                     (pl.col(\"agent_id\") == agent_mouse_id) &\n    #                     (pl.col(\"target_id\") == target_mouse_id)\n    #                 )\n    #                 .collect()\n    #             )\n    #         else:\n    #             annotation = pl.DataFrame()\n\n    #         label_frames = set()\n    #         for ann in annotation.iter_rows(named=True):\n    #             label_frames.update(range(ann[\"start_frame\"], ann[\"stop_frame\"]))\n\n    #         label = index.select(\n    #             pl.col(\"video_frame\")\n    #             .is_in(label_frames)\n    #             .cast(pl.Int8)\n    #             .alias(\"label\")\n    #         )\n\n    #         if label[\"label\"].sum() == 0:\n    #             continue\n\n    #         index_list.append(index)\n    #         feature_list.append(feature)\n    #         label_list.append(label[\"label\"])\n\n    #     if not index_list:\n    #         continue\n\n    #     indices = pl.concat(index_list)\n    #     features = pl.concat(feature_list)\n    #     labels = pl.concat(label_list)\n        \n    #     if labels.min() == labels.max():\n    #         continue\n\n    #     print(f\"Samples: {len(indices)}\")\n    #     print(f\"Positive: {labels.sum()}\")\n    #     print(f\"Features: {len(features.columns)}\")\n        \n    #     train_validate_2(lab_id, behavior, indices, features, labels, model1_dict)\n        \n    #     del indices, features, labels\n    #     gc.collect()\n    \n    del model1_dict\n    gc.collect()","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.257Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def robustify(submission: pl.DataFrame, dataset: pl.DataFrame, train_test: str = \"train\"):\n    traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{train_test}_tracking\"\n\n    old = submission\n    submission = submission.filter(pl.col(\"start_frame\") < pl.col(\"stop_frame\"))\n    if len(submission) != len(old):\n        print(\"ERROR: Dropped frames with start >= stop\")\n\n    old = submission\n    cleaned_groups = []\n\n    for _, group in submission.group_by(\"video_id\", \"agent_id\", \"target_id\"):\n        group = group.sort(\"start_frame\")\n\n        mask = np.ones(len(group), dtype=bool)\n        last_stop = -1\n\n        for i, row in enumerate(group.rows(named=True)):\n            if row[\"start_frame\"] < last_stop:\n                mask[i] = False\n            else:\n                last_stop = row[\"stop_frame\"]\n\n        cleaned_groups.append(group.filter(pl.Series(\"mask\", mask)))\n\n    submission = pl.concat(cleaned_groups)\n\n    if len(submission) != len(old):\n        print(\"ERROR: Dropped duplicate/overlap frames\")\n\n    s_list = []\n\n    submission_videos = set(submission.get_column(\"video_id\").to_list())\n\n    for r in dataset.rows(named=True):\n        lab_id = r[\"lab_id\"]\n        video_id = r[\"video_id\"]\n        labeled = r[\"behaviors_labeled\"]\n\n        if labeled is None:\n            continue\n\n        if video_id in submission_videos:\n            continue\n\n        if isinstance(labeled, str):\n            continue\n\n        print(f\"Video {video_id} has no predictions.\")\n\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        vid = pd.read_parquet(path)\n\n        behaviors = json.loads(labeled)\n        behaviors = sorted({b.replace(\"'\", \"\") for b in behaviors})\n        behaviors = [x.split(\",\") for x in behaviors]\n        behaviors = pd.DataFrame(behaviors, columns=[\"agent\", \"target\", \"action\"])\n\n        start = vid.video_frame.min()\n        stop = vid.video_frame.max() + 1\n\n        grouped = behaviors.groupby([\"agent\", \"target\"])\n\n        for (agent, target), acts in grouped:\n            batch_len = int(np.ceil((stop - start) / len(acts)))\n\n            for i, act in enumerate(acts.itertuples(index=False)):\n                st = start + i * batch_len\n                sp = min(st + batch_len, stop)\n                s_list.append((video_id, agent, target, act.action, st, sp))\n\n    if s_list:\n        print(\"ERROR: Filled empty videos\")\n        fill_df = pd.DataFrame(\n            s_list,\n            columns=[\"video_id\", \"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"]\n        )\n        submission = pl.concat([submission, pl.from_pandas(fill_df)])\n\n    return submission","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.257Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == \"SUBMIT\":\n    test_dataframe = pl.read_csv(\"/kaggle/input/MABe-mouse-behavior-detection/test.csv\")\n    test_behavior_dataframe = (\n        test_dataframe\n        .filter(pl.col(\"behaviors_labeled\").is_not_null())\n        .select(\n            pl.col(\"lab_id\"),\n            pl.col(\"video_id\"),\n            pl.col(\"behaviors_labeled\")\n                .map_elements(eval, return_dtype=pl.List(pl.Utf8))\n                .alias(\"behaviors_labeled_list\"),\n        )\n        .explode(\"behaviors_labeled_list\")\n        .rename({\"behaviors_labeled_list\": \"behaviors_labeled_element\"})\n        .select(\n            pl.col(\"lab_id\"),\n            pl.col(\"video_id\"),\n            pl.col(\"behaviors_labeled_element\")\n                .str.split(\",\")\n                .list[0]\n                .str.replace_all(\"'\", \"\")\n                .alias(\"agent\"),\n            pl.col(\"behaviors_labeled_element\")\n                .str.split(\",\")\n                .list[1]\n                .str.replace_all(\"'\", \"\")\n                .alias(\"target\"),\n            pl.col(\"behaviors_labeled_element\")\n                .str.split(\",\")\n                .list[2]\n                .str.replace_all(\"'\", \"\")\n                .alias(\"behavior\"),\n        )\n    )\n    \n    test_self_behavior_dataframe = test_behavior_dataframe.filter(\n        pl.col(\"behavior\").is_in(SELF_ACTIONS)\n    )\n    \n    test_pair_behavior_dataframe = test_behavior_dataframe.filter(\n        pl.col(\"behavior\").is_in(PAIR_ACTIONS)\n    )","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.257Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == \"SUBMIT\":\n\n    self_features_dir = \"self_features\"\n    pair_features_dir = \"pair_features\"\n    \n    os.makedirs(self_features_dir, exist_ok=True)\n    os.makedirs(pair_features_dir, exist_ok=True)\n    \n    rows = test_dataframe.rows(named=True)\n    \n    for row in tqdm(rows, total=len(rows)):\n        lab_id = row[\"lab_id\"]\n        video_id = row[\"video_id\"]\n        \n        tracking_path = f\"/kaggle/input/MABe-mouse-behavior-detection/test_tracking/{lab_id}/{video_id}.parquet\"\n        tracking = pl.read_parquet(tracking_path)\n        \n        self_features = make_self_features(metadata=row, tracking=tracking)\n        pair_features = make_pair_features(metadata=row, tracking=tracking)\n        \n        self_features.write_parquet(f\"{self_features_dir}/{video_id}.parquet\")\n        pair_features.write_parquet(f\"{pair_features_dir}/{video_id}.parquet\")","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.257Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if STATUS == \"SUBMIT\":\n\n    model1_map = {}\n    model2_map = {}\n    for lab_id in os.listdir('/kaggle/input/mlgbver2/results'):\n        for act in os.listdir(f'/kaggle/input/mlgbver2/results/{lab_id}'):\n            if 'models_1' in os.listdir(f'/kaggle/input/mlgbver2/results/{lab_id}/{act}'):\n                model1 = lgb.Booster(model_file=f'/kaggle/input/mlgbver2/results/{lab_id}/{act}/models_1/model_l.txt')\n                with open(f'/kaggle/input/mlgbver2/results/{lab_id}/{act}/models_1/threshold_l.txt', 'r', encoding='utf-8') as f:\n                    threshold = float(f.read())\n                if lab_id not in model1_map:\n                    model1_map[lab_id] = {}\n                    \n                model1_map[lab_id][act] = (model1, threshold)\n            # if 'models_2' in os.listdir(f'/kaggle/input/mlgbver2/results/{lab_id}/{act}'):\n            #     model2 = lgb.Booster(model_file=f'/kaggle/input/mlgbver2/results/{lab_id}/{act}/models_2/model_l.txt')\n            #     with open(f'/kaggle/input/mlgbver2/results/{lab_id}/{act}/models_2/threshold_l.txt', 'r', encoding='utf-8') as f:\n            #         threshold = float(f.read())\n            #     with open(f'/kaggle/input/mlgbver2/results/{lab_id}/{act}/models_1/feature_names.json', 'r', encoding='utf-8') as f:\n            #         feature_names = json.load(f)\n            #     if lab_id not in model2_map:\n            #         model2_map[lab_id] = {}\n            #     model2_map[lab_id][act] = (model2, threshold, feature_names)\n    \n    group_submissions = []\n    groups = test_behavior_dataframe.group_by(\"lab_id\", \"video_id\", \"agent\", \"target\", maintain_order=True)\n\n    for (lab_id, video_id, agent, target), group in tqdm(groups, total=len(list(groups))):\n        agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n        target_mouse_id = -1 if target == \"self\" else int(re.search(r\"mouse(\\d+)\", target).group(1))\n        \n        if target == \"self\":\n            index = (\n                pl.scan_parquet(f\"self_features/{video_id}.parquet\")\n                .filter(pl.col(\"agent_mouse_id\") == agent_mouse_id)\n                .select(INDEX_COLS)\n                .collect()\n            )\n            feature = (\n                pl.scan_parquet(f\"self_features/{video_id}.parquet\")\n                .filter(pl.col(\"agent_mouse_id\") == agent_mouse_id)\n                .select(pl.exclude(INDEX_COLS))\n                .collect()\n            )\n        else:\n            index = (\n                pl.scan_parquet(f\"pair_features/{video_id}.parquet\")\n                .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id) & \n                        (pl.col(\"target_mouse_id\") == target_mouse_id))\n                .select(INDEX_COLS)\n                .collect()\n            )\n            feature = (\n                pl.scan_parquet(f\"pair_features/{video_id}.parquet\")\n                .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id) & \n                        (pl.col(\"target_mouse_id\") == target_mouse_id))\n                .select(pl.exclude(INDEX_COLS))\n                .collect()\n            )\n        \n        prediction_dataframe = index.clone()\n        \n        for row in group.rows(named=True):\n            behavior = row[\"behavior\"]\n            predictions = []\n            prediction_labels = []\n\n            try:\n                model, thr = model1_map[lab_id][behavior]\n                feature_array = feature.to_numpy()\n                model_predictions = model.predict(feature_array)\n                predictions.append(model_predictions)\n                prediction_labels.append((model_predictions >= thr).astype(np.int8))\n            except:\n                continue\n                \n            \n            # Ghi kết quả prediction vào dataframe tạm\n            if predictions:\n                prediction_dataframe = prediction_dataframe.with_columns(\n                    *[\n                        pl.Series(name=f\"{behavior}_{fold}\", \n                                 values=predictions[fold] * prediction_labels[fold], \n                                 dtype=pl.Float32)\n                        for fold in range(len(predictions))\n                    ]\n                )\n        \n        cols = prediction_dataframe.select(pl.exclude(INDEX_COLS)).columns\n        if not cols:\n            tqdm.write(f\"Warning: No predictions for {lab_id}, {video_id}, {agent}, {target}\")\n            continue\n        \n        prediction_labels_dataframe = prediction_dataframe.with_columns(\n            pl.struct(pl.col(cols))\n            .map_elements(\n                lambda row: \"none\" if sum(row.values()) == 0 else cols[np.argmax(list(row.values()))].split(\"_\")[0],\n                return_dtype=pl.String,\n            )\n            .alias(\"prediction\")\n        ).select(INDEX_COLS + [\"prediction\"])\n        \n        group_submission = (\n            prediction_labels_dataframe\n            .filter(pl.col(\"prediction\") != pl.col(\"prediction\").shift(1))\n            .with_columns(pl.col(\"video_frame\").shift(-1).alias(\"stop_frame\"))\n            .filter(pl.col(\"prediction\") != \"none\")\n            .select(\n                pl.col(\"video_id\"),\n                (\"mouse\" + pl.col(\"agent_mouse_id\").cast(str)).alias(\"agent_id\"),\n                pl.when(pl.col(\"target_mouse_id\") == -1)\n                .then(pl.lit(\"self\"))\n                .otherwise(\"mouse\" + pl.col(\"target_mouse_id\").cast(str))\n                .alias(\"target_id\"),\n                pl.col(\"prediction\").alias(\"action\"),\n                pl.col(\"video_frame\").alias(\"start_frame\"),\n                pl.col(\"stop_frame\"),\n            )\n        )\n        \n        group_submissions.append(group_submission)\n    \n    if group_submissions:\n        submission = pl.concat(group_submissions, how=\"vertical\").sort(\n            \"video_id\", \"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"\n        )\n        \n        submission = robustify(submission, test_dataframe, train_test=\"test\")\n        \n        submission.with_row_index(\"row_id\").write_csv(\"submission.csv\")\n    else:\n        # Fallback tạo file rỗng nếu không có dự đoán nào\n        print(\"No predictions found. Creating empty submission.\")\n        pl.DataFrame(schema={\n            \"row_id\": pl.Int64, \"video_id\": pl.Int32, \"agent_id\": pl.Utf8, \n            \"target_id\": pl.Utf8, \"action\": pl.Utf8, \"start_frame\": pl.Int32, \"stop_frame\": pl.Int32\n        }).write_csv(\"submission.csv\")","metadata":{"execution":{"execution_failed":"2025-12-12T17:58:04.257Z"},"trusted":true},"outputs":[],"execution_count":null}]}