{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13652703,"sourceType":"datasetVersion","datasetId":8679438},{"sourceId":13924619,"sourceType":"datasetVersion","datasetId":8873322},{"sourceId":13924780,"sourceType":"datasetVersion","datasetId":8873425},{"sourceId":279806245,"sourceType":"kernelVersion"},{"sourceId":281545678,"sourceType":"kernelVersion"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# SECTION 1 — SETUP & IMPORTS (KAGGLE VERSION)\n# Clean unified pipeline for the MABe Challenge\n# ============================================================\n\nimport os\nimport gc\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\nimport joblib\n\n# ML Models (✔️ NOW INCLUDING LightGBM + XGBoost)\nimport lightgbm as lgb\nimport xgboost as xgb\n\nimport matplotlib.pyplot as plt\n\n# Clean output\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\n# Global reproducibility\nSEED = 42\nnp.random.seed(SEED)\n\nprint(\"✔️ Kaggle imports loaded successfully.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:43.737391Z","iopub.execute_input":"2025-11-30T06:07:43.737628Z","iopub.status.idle":"2025-11-30T06:07:51.058637Z","shell.execute_reply.started":"2025-11-30T06:07:43.737590Z","shell.execute_reply":"2025-11-30T06:07:51.057964Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 2 — CONFIGURATION (KAGGLE VERSION)\n# Dataset root and model settings\n# ============================================================\n\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier\n\n# ------------------------------------------------\n# ✔ Correct Kaggle dataset root\n# ------------------------------------------------\nDATA_ROOT = Path(\"/kaggle/input/mabe-mouse-behavior-detection\")\n\nTRAIN_ANN_DIR = DATA_ROOT / \"train_annotation\"\nTRAIN_TRACK_DIR = DATA_ROOT / \"train_tracking\"\n\n# ------------------------------------------------\n# IMPORTANT:\n# Build a mapping: video_id (int) → folder_name (e.g. \"AdaptableSnail\")\n# ------------------------------------------------\nVIDEO_TO_FOLDER = {}\n\nfor folder in sorted(os.listdir(TRAIN_ANN_DIR)):\n    folder_path = TRAIN_ANN_DIR / folder\n    for fname in os.listdir(folder_path):\n        vid = int(Path(fname).stem)\n        VIDEO_TO_FOLDER[vid] = folder\n\n# Ensure mapping is not empty\nassert len(VIDEO_TO_FOLDER) > 0, \"❌ Could not build VIDEO_TO_FOLDER mapping!\"\nprint(f\"✔️ VIDEO_TO_FOLDER mapping built: {len(VIDEO_TO_FOLDER)} videos found.\")\n\n# ------------------------------------------------\n# Feature extraction settings\n# ------------------------------------------------\nWINDOW_SIZE = 15        # annotation window\nSLIDE_WINDOW = 15       # test sliding window\nSLIDE_STEP = 5\n\n# ------------------------------------------------\n# Output Directory\n# ------------------------------------------------\nOUTPUT_DIR = Path(\"./outputs_clean_pipeline\")\nOUTPUT_DIR.mkdir(exist_ok=True, parents=True)\n\n# ------------------------------------------------\n# MODEL SELECTION\n# ------------------------------------------------\nUSE_RANDOM_FOREST = True\nUSE_EXTRATREES = True\nUSE_LIGHTGBM = True\nUSE_XGBOOST = True\n\nRF_PARAMS = {\n    \"n_estimators\": 300,\n    \"max_depth\": None,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n}\n\nEXTRA_PARAMS = {\n    \"n_estimators\": 400,\n    \"max_depth\": None,\n    \"bootstrap\": False,\n    \"n_jobs\": -1,\n    \"random_state\": 42,\n}\n\n# ------------------------------------------------\n# LightGBM Parameters\n# ------------------------------------------------\nLGBM_PARAMS = {\n    \"objective\": \"multiclass\",   # will fill num_class later\n    \"learning_rate\": 0.05,\n    \"n_estimators\": 300,\n    \"num_leaves\": 64,\n    \"max_depth\": -1,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"random_state\": 42,\n    \"verbosity\": -1,\n}\n\n# ------------------------------------------------\n# XGBoost Parameters\n# ------------------------------------------------\nXGB_PARAMS = {\n    \"objective\": \"multi:softprob\",\n    \"learning_rate\": 0.05,\n    \"max_depth\": 8,\n    \"n_estimators\": 300,\n    \"subsample\": 0.8,\n    \"colsample_bytree\": 0.8,\n    \"reg_alpha\": 0.1,\n    \"reg_lambda\": 1.0,\n    \"eval_metric\": \"mlogloss\",\n    \"verbosity\": 0,\n    \"n_jobs\": -1,\n}\n\nprint(\"✔️ Kaggle configuration loaded.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.059805Z","iopub.execute_input":"2025-11-30T06:07:51.060238Z","iopub.status.idle":"2025-11-30T06:07:51.557406Z","shell.execute_reply.started":"2025-11-30T06:07:51.060219Z","shell.execute_reply":"2025-11-30T06:07:51.556595Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 3 — DATA LOADING (KAGGLE, MEMORY-SAFE & CORRECT)\n# ============================================================\n\ndef load_annotations_train(data_root):\n    \"\"\"\n    Load ALL training annotation parquet files.\n    Each annotation file corresponds to one video.\n    Folder structure:\n        train_annotation/<folder>/<video_id>.parquet\n    \"\"\"\n    ann_root = Path(data_root) / \"train_annotation\"\n    dfs = []\n\n    print(\"🔍 Scanning annotation folders...\")\n\n    for folder in sorted(ann_root.iterdir()):\n        if not folder.is_dir():\n            continue\n\n        for f in folder.glob(\"*.parquet\"):\n            df = pd.read_parquet(f)\n\n            # Numeric video ID from filename\n            video_id = int(f.stem)\n            df[\"video_id\"] = video_id\n\n            dfs.append(df)\n\n    if not dfs:\n        raise ValueError(\"❌ No annotation parquet files found!\")\n\n    full_df = pd.concat(dfs, ignore_index=True)\n    return full_df\n\n\n\ndef load_tracking_file(video_id, split=\"train\", data_root=DATA_ROOT):\n    \"\"\"\n    Load a tracking parquet file and pivot keypoints\n    so each frame has ONE row per mouse:\n      mouse_id, video_frame, x_bodypart, y_bodypart, ...\n    \"\"\"\n    track_root = Path(data_root) / f\"{split}_tracking\"\n    matches = list(track_root.rglob(f\"{video_id}.parquet\"))\n\n    if not matches:\n        raise FileNotFoundError(f\"❌ Tracking parquet for video_id={video_id} not found.\")\n\n    df = pd.read_parquet(matches[0])\n\n    # -------------------------------------\n    # Normalize column names\n    # -------------------------------------\n    if \"frame_idx\" in df.columns:\n        df.rename(columns={\"frame_idx\": \"video_frame\"}, inplace=True)\n\n    if \"frame\" in df.columns and \"video_frame\" not in df.columns:\n        df.rename(columns={\"frame\": \"video_frame\"}, inplace=True)\n\n    if \"mouse\" in df.columns:\n        df.rename(columns={\"mouse\": \"mouse_id\"}, inplace=True)\n\n    df[\"video_frame\"] = df[\"video_frame\"].astype(int)\n\n    # -------------------------------------\n    # 🔥 Pivot: bodypart rows → wide keypoints per frame\n    # -------------------------------------\n    df_pivot = df.pivot_table(\n        index=[\"mouse_id\", \"video_frame\"],\n        columns=\"bodypart\",\n        values=[\"x\", \"y\"],\n        aggfunc=\"first\",\n    )\n\n    # Flatten x/y column names → x_head, y_head, x_nose, y_nose, etc.\n    df_pivot.columns = [f\"{coord}_{bp}\" for coord, bp in df_pivot.columns]\n\n    # Reset index → normal dataframe\n    df_pivot = df_pivot.reset_index()\n\n    # Add video ID\n    df_pivot[\"video_id\"] = video_id\n\n    # Fill missing keypoints\n    df_pivot = df_pivot.fillna(0)\n\n    return df_pivot\n\n\n\ndef list_train_video_ids(data_root=DATA_ROOT):\n    \"\"\"\n    Return all train video IDs by searching nested train_tracking folders.\n    \"\"\"\n    track_dir = Path(data_root) / \"train_tracking\"\n    files = track_dir.rglob(\"*.parquet\")\n    video_ids = sorted({int(f.stem) for f in files})\n\n    if len(video_ids) == 0:\n        print(\"⚠️ No train video IDs found! Check path.\")\n\n    return video_ids\n\n\n\ndef list_test_video_ids(data_root=DATA_ROOT):\n    \"\"\"\n    Return all test video IDs by searching nested test_tracking folders.\n    \"\"\"\n    test_dir = Path(data_root) / \"test_tracking\"\n    files = test_dir.rglob(\"*.parquet\")\n    video_ids = sorted({int(f.stem) for f in files})\n\n    if len(video_ids) == 0:\n        print(\"⚠️ No test video IDs found! Check path.\")\n\n    return video_ids\n\n\nprint(\"✔️ Data loader (Kaggle + memory-safe + folder-aware) ready.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.558196Z","iopub.execute_input":"2025-11-30T06:07:51.558406Z","iopub.status.idle":"2025-11-30T06:07:51.570264Z","shell.execute_reply.started":"2025-11-30T06:07:51.558389Z","shell.execute_reply":"2025-11-30T06:07:51.569399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 4A — FEATURE UTILITIES (FIXED FOR MABe)\n# ============================================================\n\ndef detect_xy_columns(df):\n    \"\"\"\n    Detect columns that represent x/y coordinates in MABe tracking.\n\n    Expected valid patterns:\n        x, y\n        x_head, y_head\n        x_tail, y_tail\n        nose_x, nose_y\n\n    Falls back to the first matching x_*/y_* pattern.\n    If nothing is found, tries plain 'x' and 'y'.\n    \"\"\"\n    \n    # Explicit known patterns\n    xy_cols = [\n        (c.lower(), c) for c in df.columns\n        if c.lower() in [\n            \"x\", \"y\",\n            \"x_head\", \"y_head\",\n            \"x_tail\", \"y_tail\",\n            \"nose_x\", \"nose_y\"\n        ]\n    ]\n\n    if xy_cols:\n        # return first match in original column name format\n        return xy_cols[0][1], xy_cols[1][1] if len(xy_cols) > 1 else xy_cols[0][1]\n\n    # Generic fallback: first x_*, y_*\n    generic_x = [c for c in df.columns if c.lower().startswith(\"x_\")]\n    generic_y = [c for c in df.columns if c.lower().startswith(\"y_\")]\n\n    if generic_x and generic_y:\n        return generic_x[0], generic_y[0]\n\n    # Basic fallback: raw x and y\n    if \"x\" in df.columns and \"y\" in df.columns:\n        return \"x\", \"y\"\n\n    raise ValueError(\n        f\"❌ Could not detect XY columns in tracking df.\\n\"\n        f\"Available columns: {list(df.columns)}\"\n    )\n\n\n\ndef compute_stats(arr):\n    \"\"\"\n    Compute basic statistical summary features from a 1D numeric array.\n    \n    Returns a 7-element vector:\n        [mean, std, min, max, p25, p50, p75]\n\n    Handles empty inputs gracefully by returning zeros.\n    \"\"\"\n    if arr is None or len(arr) == 0:\n        return [0, 0, 0, 0, 0, 0, 0]\n\n    arr = np.array(arr, dtype=np.float32)\n\n    return [\n        float(np.mean(arr)),\n        float(np.std(arr)),\n        float(np.min(arr)),\n        float(np.max(arr)),\n        float(np.percentile(arr, 25)),\n        float(np.percentile(arr, 50)),\n        float(np.percentile(arr, 75)),\n    ]\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.571206Z","iopub.execute_input":"2025-11-30T06:07:51.571494Z","iopub.status.idle":"2025-11-30T06:07:51.587038Z","shell.execute_reply.started":"2025-11-30T06:07:51.571469Z","shell.execute_reply":"2025-11-30T06:07:51.586301Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 4B — TRAINING FEATURES (video-by-video loading)\n# FINAL FIXED VERSION — SAFE FOR SPARSE / KEYPOINT TRACKING\n# ============================================================\n\ndef generate_training_features(annotations_df, window_size=WINDOW_SIZE, data_root=DATA_ROOT):\n    \"\"\"\n    Safe feature generation for sparse / keypoint tracking.\n    \n    Works even if:\n      - annotations reference frames not present in tracking\n      - tracking has missing or irregular frame numbering\n      - tracking is keypoint-based (multiple bodyparts per frame)\n    \"\"\"\n\n    print(\"🔧 Generating TRAINING features (video-by-video)...\")\n\n    ann_by_video = annotations_df.groupby(\"video_id\")\n    X_rows = []\n    y_rows = []\n\n    for video_id, ann_group in tqdm(ann_by_video, desc=\"Videos\"):\n\n        # ----------------------------------------------------\n        # 1. Load tracking (pivoted → wide format)\n        # ----------------------------------------------------\n        try:\n            track_df = load_tracking_file(video_id, split=\"train\", data_root=data_root)\n        except Exception as e:\n            print(f\"⚠️ Cannot load tracking for video {video_id}: {e}\")\n            continue\n\n        # Detect XY columns\n        try:\n            x_col, y_col = detect_xy_columns(track_df)\n        except Exception:\n            print(f\"⚠️ Cannot detect XY columns for video {video_id}\")\n            continue\n\n        # Sort and index by (mouse_id, frame)\n        track_df = track_df.sort_values([\"mouse_id\", \"video_frame\"])\n        idx = track_df.set_index([\"mouse_id\", \"video_frame\"])\n\n        # ----------------------------------------------------\n        # 2. Process annotation events for this video\n        # ----------------------------------------------------\n        for _, row in ann_group.iterrows():\n\n            agent = row[\"agent_id\"]\n            target = row[\"target_id\"]\n\n            start = int(row[\"start_frame\"])\n            stop  = int(row[\"stop_frame\"])\n\n            # Expanded window around event\n            s = max(0, start - window_size)\n            e = stop + window_size\n\n            # ----------------------------\n            # Agent frames (safe extraction)\n            # ----------------------------\n            try:\n                agent_sub = idx.loc[agent]\n                agent_frames = agent_sub.index.values\n            except KeyError:\n                continue\n\n            valid_agent_frames = agent_frames[(agent_frames >= s) & (agent_frames <= e)]\n            if len(valid_agent_frames) == 0:\n                continue\n\n            valid_agent_frames = np.intersect1d(valid_agent_frames, agent_frames)\n            if len(valid_agent_frames) == 0:\n                continue\n\n            at = agent_sub.loc[valid_agent_frames]\n\n            # ----------------------------\n            # Target frames (safe extraction)\n            # ----------------------------\n            try:\n                target_sub = idx.loc[target]\n                target_frames = target_sub.index.values\n            except KeyError:\n                continue\n\n            valid_target_frames = target_frames[(target_frames >= s) & (target_frames <= e)]\n            if len(valid_target_frames) == 0:\n                continue\n\n            valid_target_frames = np.intersect1d(valid_target_frames, target_frames)\n            if len(valid_target_frames) == 0:\n                continue\n\n            tt = target_sub.loc[valid_target_frames]\n\n            # ----------------------------\n            # Align by overlapping frames\n            # ----------------------------\n            merged = (\n                at.reset_index()\n                  .merge(tt.reset_index(), on=\"video_frame\", suffixes=(\"_a\", \"_t\"))\n                  .fillna(0)\n            )\n\n            if merged.empty:\n                continue\n\n            # ----------------------------------------------------\n            # Compute geometric & motion features\n            # ----------------------------------------------------\n            ax = merged[f\"{x_col}_a\"].astype(np.float32)\n            ay = merged[f\"{y_col}_a\"].astype(np.float32)\n            tx = merged[f\"{x_col}_t\"].astype(np.float32)\n            ty = merged[f\"{y_col}_t\"].astype(np.float32)\n\n            dx = ax - tx\n            dy = ay - ty\n            dist = np.sqrt(dx * dx + dy * dy)\n\n            vx = np.diff(ax, prepend=ax.iloc[0])\n            vy = np.diff(ay, prepend=ay.iloc[0])\n            speed = np.sqrt(vx * vx + vy * vy)\n\n            # Final feature vector (28 dims)\n            feats = (\n                compute_stats(dx) +\n                compute_stats(dy) +\n                compute_stats(dist) +\n                compute_stats(speed)\n            )\n\n            X_rows.append(feats)\n            y_rows.append(row[\"action\"])\n\n        # Memory cleanup per video\n        del track_df\n        gc.collect()\n\n    # Convert to DataFrame / Series\n    X = pd.DataFrame(X_rows)\n    y = pd.Series(y_rows, name=\"action\")\n\n    print(f\"✔️ Training features generated: X={X.shape}, y={len(y)}\")\n    return X, y\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.589105Z","iopub.execute_input":"2025-11-30T06:07:51.589377Z","iopub.status.idle":"2025-11-30T06:07:51.605617Z","shell.execute_reply.started":"2025-11-30T06:07:51.589360Z","shell.execute_reply":"2025-11-30T06:07:51.604989Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 5 — MODEL DEFINITION (RF + ExtraTrees + LightGBM + XGBoost)\n# With correct LabelEncoder handling\n# ============================================================\n\nfrom sklearn.preprocessing import LabelEncoder\n\n# -------------------------------------------\n# Global LabelEncoder (shared by all models)\n# -------------------------------------------\nlabel_encoder = LabelEncoder()\n\n\n# ============================================================\n# Base Wrapper (common utilities)\n# ============================================================\nclass BaseModel:\n    def predict(self, X):\n        \"\"\"Return encoded predictions.\"\"\"\n        return self.model.predict(X)\n\n    def predict_labels(self, X):\n        \"\"\"Return string (original) action labels.\"\"\"\n        preds = self.model.predict(X)\n        return label_encoder.inverse_transform(preds)\n\n    def predict_proba(self, X):\n        \"\"\"Every model must expose probability predictions.\"\"\"\n        return self.model.predict_proba(X)\n\n\n# ============================================================\n# RandomForest Wrapper\n# ============================================================\nclass RFModel(BaseModel):\n    def __init__(self, params):\n        params = params.copy()\n        params.setdefault(\"class_weight\", \"balanced\")\n        self.model = RandomForestClassifier(**params)\n\n    def fit(self, X, y):\n        self.model.fit(X, y)\n\n\n# ============================================================\n# ExtraTrees Wrapper\n# ============================================================\nclass ExtraTreesModel(BaseModel):\n    def __init__(self, params):\n        params = params.copy()\n        params.setdefault(\"class_weight\", \"balanced\")\n        self.model = ExtraTreesClassifier(**params)\n\n    def fit(self, X, y):\n        self.model.fit(X, y)\n\n\n# ============================================================\n# LightGBM Wrapper\n# ============================================================\nclass LGBMModel(BaseModel):\n    def __init__(self, params):\n        params = params.copy()\n        params.setdefault(\"objective\", \"multiclass\")\n        params.setdefault(\"num_class\", NUM_CLASSES)\n        params.setdefault(\"verbosity\", -1)\n        params.setdefault(\"random_state\", 42)\n        self.model = lgb.LGBMClassifier(**params)\n\n    def fit(self, X, y):\n        self.model.fit(X, y)\n\n\n# ============================================================\n# XGBoost Wrapper\n# ============================================================\nclass XGBModel(BaseModel):\n    def __init__(self, params):\n        params = params.copy()\n        params.setdefault(\"objective\", \"multi:softprob\")\n        params.setdefault(\"num_class\", NUM_CLASSES)\n        params.setdefault(\"eval_metric\", \"mlogloss\")\n        params.setdefault(\"verbosity\", 0)\n        params.setdefault(\"n_jobs\", -1)\n        self.model = xgb.XGBClassifier(**params)\n\n    def fit(self, X, y):\n        self.model.fit(X, y)\n\n\n# ============================================================\n# Factory: initialize selected models (from config)\n# ============================================================\ndef get_models():\n    models = []\n\n    if USE_RANDOM_FOREST:\n        print(\"🌲 Initializing RandomForest...\")\n        models.append((\"rf\", RFModel(RF_PARAMS)))\n\n    if USE_EXTRATREES:\n        print(\"🌳 Initializing ExtraTrees...\")\n        models.append((\"extra\", ExtraTreesModel(EXTRA_PARAMS)))\n\n    if USE_LIGHTGBM:\n        print(\"💡 Initializing LightGBM...\")\n        models.append((\"lgbm\", LGBMModel(LGBM_PARAMS)))\n\n    if USE_XGBOOST:\n        print(\"🔥 Initializing XGBoost...\")\n        models.append((\"xgb\", XGBModel(XGB_PARAMS)))\n\n    if not models:\n        raise ValueError(\"❌ No models selected in Section 2 config.\")\n\n    return models\n\n\n# ============================================================\n# Ensemble utility (probability averaging)\n# ============================================================\ndef ensemble_probabilities(list_of_probas):\n    \"\"\"\n    Average probability matrices from multiple models.\n    Ensures consistent class order (same NUM_CLASSES) across models.\n    \"\"\"\n    arr = np.stack(list_of_probas, axis=0)  # shape: (num_models, N, num_classes)\n    return np.mean(arr, axis=0)\n\n\nprint(\"✔️ Models ready (RF + ExtraTrees + LightGBM + XGBoost).\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.606455Z","iopub.execute_input":"2025-11-30T06:07:51.606711Z","iopub.status.idle":"2025-11-30T06:07:51.622896Z","shell.execute_reply.started":"2025-11-30T06:07:51.606692Z","shell.execute_reply":"2025-11-30T06:07:51.622032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 6 — TRAINING PIPELINE (Stable Kaggle Version)\n# ============================================================\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, f1_score, classification_report\nfrom sklearn.exceptions import UndefinedMetricWarning\nimport warnings\n\nwarnings.filterwarnings(\"ignore\", category=UndefinedMetricWarning)\n\n# ---------------------------\n# 1. Load training annotations\n# ---------------------------\nprint(\"📥 Loading training annotations...\")\nannotations_train = load_annotations_train(DATA_ROOT)\nprint(\"✔️ Annotations:\", annotations_train.shape)\n\n\n# ---------------------------\n# 2. Generate training features\n# ---------------------------\nprint(\"\\n🔧 Generating TRAINING features...\")\nX_train_raw, y_train_raw = generate_training_features(\n    annotations_train,\n    window_size=WINDOW_SIZE,\n    data_root=DATA_ROOT\n)\n\n# fix NaNs\nX_train_raw = X_train_raw.fillna(0)\n\nif X_train_raw.empty:\n    raise ValueError(\"❌ No training features produced. Likely tracking lookup mismatch.\")\n\nprint(\"✔️ X shape:\", X_train_raw.shape)\nprint(\"✔️ y count:\", len(y_train_raw))\n\n\n# ---------------------------\n# 3. Encode labels\n# ---------------------------\nprint(\"\\n🔧 Encoding labels...\")\ny_encoded = label_encoder.fit_transform(y_train_raw)\n\nclasses = list(label_encoder.classes_)\nNUM_CLASSES = len(classes)\n\nprint(\"✔️ Classes:\", classes)\nprint(\"✔️ NUM_CLASSES =\", NUM_CLASSES)\n\n# Inject NUM_CLASSES into LGBM & XGB params (required for multiclass)\nLGBM_PARAMS[\"num_class\"] = NUM_CLASSES\nXGB_PARAMS[\"num_class\"] = NUM_CLASSES\n\n\n# ---------------------------\n# 4. Train/Validation Split\n# ---------------------------\nprint(\"\\n🔧 Splitting train/val...\")\n\ntry:\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_raw,\n        y_encoded,\n        test_size=0.25,\n        random_state=42,\n        stratify=y_encoded\n    )\nexcept Exception as e:\n    print(\"⚠️ Stratified split failed:\", e)\n    print(\"➡️ Using NON-STRATIFIED split instead.\")\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_raw,\n        y_encoded,\n        test_size=0.25,\n        random_state=42\n    )\n\nprint(\"✔️ Train:\", X_train.shape)\nprint(\"✔️ Val:\", X_val.shape)\n\n\n# ---------------------------\n# 5. Initialize selected models\n# ---------------------------\nprint(\"\\n⚙️ Initializing models...\")\nmodels = get_models()\ntrained_models = {}\n\n\n# ---------------------------\n# 6. Train + Validate each model\n# ---------------------------\nfor name, model_obj in models:\n    print(f\"\\n🚀 Training: {name.upper()}\")\n\n    # Train\n    model_obj.fit(X_train, y_train)\n    trained_models[name] = model_obj\n\n    # Validate\n    val_proba = model_obj.predict_proba(X_val)\n    val_pred = np.argmax(val_proba, axis=1)\n\n    # Metrics\n    acc = accuracy_score(y_val, val_pred)\n    f1 = f1_score(y_val, val_pred, average=\"macro\", zero_division=0)\n\n    print(f\"✔️ {name} Accuracy: {acc:.4f}\")\n    print(f\"✔️ {name} Macro F1 : {f1:.4f}\")\n\n    # Classification report (safe for missing classes)\n    print(\"\\n🔍 Classification Report:\")\n    print(classification_report(\n        y_val,\n        val_pred,\n        labels=np.arange(NUM_CLASSES),\n        target_names=classes,\n        zero_division=0\n    ))\n\n    gc.collect()\n\n\n# ---------------------------\n# 7. Save all trained models + encoder\n# ---------------------------\nfor name, model_obj in trained_models.items():\n    path = OUTPUT_DIR / f\"{name}_model.pkl\"\n    joblib.dump(model_obj, path)\n    print(f\"💾 Saved model → {path}\")\n\n# Save LabelEncoder\njoblib.dump(label_encoder, OUTPUT_DIR / \"label_encoder.pkl\")\nprint(\"💾 Saved LabelEncoder → label_encoder.pkl\")\n\nprint(\"\\n🎯 Training pipeline completed successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-30T06:07:51.623745Z","iopub.execute_input":"2025-11-30T06:07:51.624000Z","execution_failed":"2025-11-30T06:11:52.020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def load_test_tracking_recursive(video_id, data_root=DATA_ROOT):\n    \"\"\"\n    Custom loader ONLY for test inference.\n    Recursively searches for <video_id>.parquet inside:\n        test_tracking/**/<video_id>.parquet\n    \n    Returns a pandas DataFrame or None if missing.\n    \"\"\"\n    root = Path(data_root) / \"test_tracking\"\n\n    # recursive search\n    files = list(root.glob(f\"**/{video_id}.parquet\"))\n\n    if not files:\n        print(f\"⚠️ No test tracking file found for video_id={video_id}\")\n        return None\n\n    try:\n        df = pd.read_parquet(files[0])\n    except Exception as e:\n        print(f\"❌ Failed to read test tracking for {video_id}: {e}\")\n        return None\n\n    # ------------------------------------------------------\n    # Normalize column names (same as training loader)\n    # ------------------------------------------------------\n    if \"frame_idx\" in df.columns:\n        df.rename(columns={\"frame_idx\": \"video_frame\"}, inplace=True)\n    if \"frame\" in df.columns and \"video_frame\" not in df.columns:\n        df.rename(columns={\"frame\": \"video_frame\"}, inplace=True)\n    if \"mouse\" in df.columns:\n        df.rename(columns={\"mouse\": \"mouse_id\"}, inplace=True)\n\n    df[\"video_frame\"] = df[\"video_frame\"].astype(int)\n    df = df.fillna(0)\n\n    return df\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-30T06:11:52.020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# SECTION 7 — TEST INFERENCE (Sliding-Window)\n# ============================================================\n\ndef generate_test_features_for_video(video_id, slide_window=SLIDE_WINDOW, slide_step=SLIDE_STEP):\n    \"\"\"\n    Safe sliding-window feature extraction for TEST videos.\n    Handles sparse frames, missing keypoints, and irregular tracking.\n    \"\"\"\n\n    # Load test tracking\n    try:\n        track_df = load_test_tracking_recursive(video_id, data_root=DATA_ROOT)\n        if track_df is None:\n            raise FileNotFoundError(\"Returned None from loader.\")\n    except Exception as e:\n        print(f\"⚠️ Cannot load test tracking for video {video_id}: {e}\")\n        return None\n\n    # Detect XY columns\n    try:\n        x_col, y_col = detect_xy_columns(track_df)\n    except:\n        print(f\"⚠️ Cannot detect xy for test video {video_id}\")\n        return None\n\n    # Sort & index\n    track_df = track_df.sort_values([\"mouse_id\", \"video_frame\"])\n    idx = track_df.set_index([\"mouse_id\", \"video_frame\"])\n\n    all_frames = track_df[\"video_frame\"].unique()\n    if len(all_frames) == 0:\n        return None\n\n    max_frame = all_frames.max()\n    rows = []\n\n    # Test: typically mouse_id = [1,2]\n    mice = sorted(track_df[\"mouse_id\"].unique())\n    if len(mice) < 2:\n        return None\n\n    agent, target = mice[0], mice[1]\n\n    # Sliding window\n    for start in range(0, max_frame - slide_window, slide_step):\n\n        end = start + slide_window\n\n        # -------- AGENT --------\n        try:\n            agent_sub = idx.loc[agent]\n        except KeyError:\n            continue\n\n        agent_frames = agent_sub.index.values\n        valid_af = agent_frames[(agent_frames >= start) & (agent_frames <= end)]\n        if len(valid_af) == 0:\n            continue\n\n        valid_agent_df = agent_sub.loc[valid_af]\n\n        # -------- TARGET --------\n        try:\n            target_sub = idx.loc[target]\n        except KeyError:\n            continue\n\n        target_frames = target_sub.index.values\n        valid_tf = target_frames[(target_frames >= start) & (target_frames <= end)]\n        if len(valid_tf) == 0:\n            continue\n\n        valid_target_df = target_sub.loc[valid_tf]\n\n        # -------- MERGE --------\n        merged = (\n            valid_agent_df.reset_index()\n            .merge(valid_target_df.reset_index(), on=\"video_frame\",\n                   suffixes=(\"_a\", \"_t\"))\n            .fillna(0)\n        )\n\n        if merged.empty:\n            continue\n\n        ax = merged[f\"{x_col}_a\"].astype(np.float32)\n        ay = merged[f\"{y_col}_a\"].astype(np.float32)\n        tx = merged[f\"{x_col}_t\"].astype(np.float32)\n        ty = merged[f\"{y_col}_t\"].astype(np.float32)\n\n        dx = ax - tx\n        dy = ay - ty\n        dist = np.sqrt(dx * dx + dy * dy)\n\n        vx = np.diff(ax, prepend=ax.iloc[0])\n        vy = np.diff(ay, prepend=ay.iloc[0])\n        speed = np.sqrt(vx * vx + vy * vy)\n\n        feats = (\n            compute_stats(dx)\n            + compute_stats(dy)\n            + compute_stats(dist)\n            + compute_stats(speed)\n        )\n\n        rows.append((start, feats))\n\n    if len(rows) == 0:\n        return None\n\n    frame_starts, feats = zip(*rows)\n    return agent, target, frame_starts, pd.DataFrame(list(feats))\n\n\n\n# ============================================================\n# SEGMENTATION LOGIC (Convert frame predictions → segments)\n# ============================================================\n\ndef convert_to_segments(df):\n    \"\"\"\n    Convert frame-level predictions into continuous action segments.\n    Kaggle format required:\n    row_id, video_id, agent_id, target_id, action, start_frame, stop_frame\n    \"\"\"\n\n    segments = []\n    row_id = 0\n\n    for vid, group in df.groupby(\"video_id\"):\n\n        group = group.sort_values(\"start_frame\")\n\n        prev_action = None\n        seg_start = None\n        prev_frame = None\n        agent_id = None\n        target_id = None\n\n        for _, row in group.iterrows():\n            action = row[\"action\"]\n            frame = row[\"start_frame\"]\n            agent_id = row[\"agent_id\"]\n            target_id = row[\"target_id\"]\n\n            if action != prev_action:\n                # close previous segment\n                if prev_action is not None:\n                    segments.append([\n                        row_id, vid, agent_id, target_id,\n                        prev_action, seg_start, prev_frame\n                    ])\n                    row_id += 1\n\n                seg_start = frame\n                prev_action = action\n\n            prev_frame = frame\n\n        # close last segment\n        if prev_action is not None:\n            segments.append([\n                row_id, vid, agent_id, target_id,\n                prev_action, seg_start, prev_frame\n            ])\n            row_id += 1\n\n    return pd.DataFrame(\n        segments,\n        columns=[\n            \"row_id\",\n            \"video_id\",\n            \"agent_id\",\n            \"target_id\",\n            \"action\",\n            \"start_frame\",\n            \"stop_frame\"\n        ]\n    )\n\n\n\n# ============================================================\n# PREDICT TEST SET & EXPORT FINAL KAGGLE SUBMISSION\n# ============================================================\n\ndef run_test_inference():\n    print(\"\\n🚀 Running test inference...\")\n\n    test_video_files = list((DATA_ROOT / \"test_tracking\").glob(\"**/*.parquet\"))\n    print(\"📂 Found test test files:\", len(test_video_files))\n\n    test_video_ids = sorted({int(p.stem) for p in test_video_files})\n\n    outputs = []\n\n    for vid in tqdm(test_video_ids, desc=\"Test Videos\"):\n        result = generate_test_features_for_video(vid)\n\n        if result is None:\n            continue\n\n        agent, target, frame_starts, X_test = result\n        X_test = X_test.fillna(0)\n\n        # Ensemble prediction\n        model_preds = []\n        for _, model_obj in trained_models.items():\n            proba = model_obj.predict_proba(X_test)\n            model_preds.append(proba)\n\n        final_proba = ensemble_probabilities(model_preds)\n        final_labels = np.argmax(final_proba, axis=1)\n        final_actions = label_encoder.inverse_transform(final_labels)\n\n        for frame, action in zip(frame_starts, final_actions):\n            outputs.append([vid, agent, target, frame, action])\n\n    df_frames = pd.DataFrame(\n        outputs,\n        columns=[\"video_id\", \"agent_id\", \"target_id\", \"start_frame\", \"action\"]\n    ).sort_values([\"video_id\", \"start_frame\"])\n\n    submission = convert_to_segments(df_frames)\n\n    # Save required Kaggle file\n    submission.to_csv(\"/kaggle/working/submission.csv\", index=False)\n    print(\"📁 Saved Kaggle submission.csv to /kaggle/working/submission.csv\")\n\n    return submission\n\n\nprint(\"✔️ Test inference module ready.\")\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-30T06:11:52.021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub = run_test_inference()\n\nsub.head()\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-11-30T06:11:52.021Z"}},"outputs":[],"execution_count":null}]}