{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport os, sys, gc, math\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\n\n\ntry:\n    import polars as pl\n    USE_POLARS = True\n    print(\"Using polars\")\nexcept Exception as e:\n    USE_POLARS = False\n    print(\"polars not available — falling back to pandas\")\n\nINPUT_DIR = \"/kaggle/input/MABe-mouse-behavior-detection\"\nprint(\"Input exists?\", os.path.exists(INPUT_DIR))\nprint(\"Top-level files/folders:\")\nfor p in sorted(os.listdir(INPUT_DIR))[:200]:\n    print(\" \", p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:50:56.657123Z","iopub.execute_input":"2025-10-16T17:50:56.657888Z","iopub.status.idle":"2025-10-16T17:50:59.533717Z","shell.execute_reply.started":"2025-10-16T17:50:56.657862Z","shell.execute_reply":"2025-10-16T17:50:59.532557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nfrom typing import Tuple\n\ndef read_tracking_parquet(path: str):\n    \"\"\"Return pandas.DataFrame with columns:\n       ['video_frame', 'mouse_id', 'bodypart', 'x', 'y', 'likelihood' (if exists)]\"\"\"\n    if USE_POLARS:\n        df = pl.read_parquet(path)\n        df = df.to_pandas()\n    else:\n        df = pd.read_parquet(path)\n\n    expected_cols = set(df.columns)\n    \n    print(\"Loaded\", path, \"shape:\", df.shape)\n    print(df.columns.tolist()[:40])\n    return df\n\n\nfound = False\nfor folder in [\"train_tracking\", \"test_tracking\"]:\n    folder_path = os.path.join(INPUT_DIR, folder)\n    if os.path.exists(folder_path):\n        \n        for lab in sorted(os.listdir(folder_path))[:5]:\n            lab_dir = os.path.join(folder_path, lab)\n            if os.path.isdir(lab_dir):\n                files = [f for f in os.listdir(lab_dir) if f.endswith(\".parquet\")]\n                if len(files) > 0:\n                    test_path = os.path.join(lab_dir, files[0])\n                    print(\"Sample parquet:\", test_path)\n                    df_sample = read_tracking_parquet(test_path)\n                    found = True\n                    break\n    if found: break\n\nif not found:\n    print(\"No parquet found in expected folders — check INPUT_DIR contents.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:54:08.381121Z","iopub.execute_input":"2025-10-16T17:54:08.381448Z","iopub.status.idle":"2025-10-16T17:54:08.963773Z","shell.execute_reply.started":"2025-10-16T17:54:08.381424Z","shell.execute_reply":"2025-10-16T17:54:08.962865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndef pivot_tracking(df):\n    \n    cols = df.columns.tolist()\n    \n    frame_col = None\n    for c in ['video_frame','frame','frame_id','frameId']:\n        if c in df.columns:\n            frame_col = c\n            break\n    assert frame_col is not None, \"Cannot find frame column in tracking file\"\n\n    \n    df = df.rename(columns={frame_col: 'video_frame'})\n    \n    assert 'mouse_id' in df.columns and 'bodypart' in df.columns and 'x' in df.columns and 'y' in df.columns\n\n    \n    p = df.pivot_table(index='video_frame', columns=['mouse_id','bodypart'], values=['x','y'], aggfunc='first')\n    \n    p = p.sort_index(axis=1, level=[0,1])\n    return p\n\ndef compute_body_centers(pivot_df):\n    \"\"\"Given pivot df with columns ('x'/'y', mouse_id, bodypart) -> return DataFrame with per-mouse center coords\"\"\"\n    \n    cols = pivot_df.columns\n    mice = sorted(set([c[1] for c in cols]))\n    centers = pd.DataFrame(index=pivot_df.index)\n    for m in mice:\n        \n        x_cols = [c for c in cols if c[1]==m and c[0]=='x']\n        y_cols = [c for c in cols if c[1]==m and c[0]=='y']\n        if len(x_cols)==0:\n            continue\n        centers[f'mouse{m}_cx'] = pivot_df[x_cols].mean(axis=1)\n        centers[f'mouse{m}_cy'] = pivot_df[y_cols].mean(axis=1)\n    return centers\n\nif 'df_sample' in globals():\n    p = pivot_tracking(df_sample)\n    centers = compute_body_centers(p)\n    print(\"Pivot shape:\", p.shape, \"Centers shape:\", centers.shape)\n    display(centers.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T17:57:34.279302Z","iopub.execute_input":"2025-10-16T17:57:34.280359Z","iopub.status.idle":"2025-10-16T17:57:35.767277Z","shell.execute_reply.started":"2025-10-16T17:57:34.280316Z","shell.execute_reply":"2025-10-16T17:57:35.766509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef add_motion_features(centers: pd.DataFrame, prefix: str):\n    \"\"\"\n    Extract motion-related features for one mouse.\n    - السرعة، التسارع، الاتجاه، والتغير الزاوي\n    \"\"\"\n    cx = centers[f'{prefix}_cx']\n    cy = centers[f'{prefix}_cy']\n\n    \n    vx = cx.diff().fillna(0.0)\n    vy = cy.diff().fillna(0.0)\n    speed = np.sqrt(vx**2 + vy**2)\n\n    \n    ax = vx.diff().fillna(0.0)\n    ay = vy.diff().fillna(0.0)\n    accel = np.sqrt(ax**2 + ay**2)\n\n\n    heading = np.arctan2(vy, vx).fillna(0.0)\n\n    \n    heading_unwrapped = np.unwrap(heading.to_numpy())\n    ang_diff = np.abs(np.diff(heading_unwrapped, prepend=heading_unwrapped[0]))\n    ang_diff = pd.Series(ang_diff, index=centers.index)\n\n    features = pd.DataFrame({\n        f'{prefix}_vx': vx,\n        f'{prefix}_vy': vy,\n        f'{prefix}_speed': speed,\n        f'{prefix}_ax': ax,\n        f'{prefix}_ay': ay,\n        f'{prefix}_accel': accel,\n        f'{prefix}_heading': heading,\n        f'{prefix}_angdiff': ang_diff\n    }, index=centers.index)\n\n    return features\n\n\ndef add_pair_features(centers: pd.DataFrame, a: str, b: str):\n    \"\"\"\n    حساب المسافة والاتجاه بين فأرين (a و b)\n    \"\"\"\n    ax = centers[f'{a}_cx']; ay = centers[f'{a}_cy']\n    bx = centers[f'{b}_cx']; by = centers[f'{b}_cy']\n\n    rel_x = bx - ax\n    rel_y = by - ay\n    dist = np.sqrt(rel_x**2 + rel_y**2)\n    angle = np.arctan2(rel_y, rel_x)\n\n    pair_df = pd.DataFrame({\n        f'{a}_{b}_dx': rel_x,\n        f'{a}_{b}_dy': rel_y,\n        f'{a}_{b}_dist': dist,\n        f'{a}_{b}_angle': angle\n    }, index=centers.index)\n    return pair_df\n\n\n# 🔹 تجربة عملية لو centers موجود\nif 'centers' in globals() and centers.shape[1] >= 4:\n    f1 = add_motion_features(centers, 'mouse1')\n    f2 = add_motion_features(centers, 'mouse2')\n    fp = add_pair_features(centers, 'mouse1', 'mouse2')\n    display(pd.concat([f1.iloc[:10], f2.iloc[:10], fp.iloc[:10]], axis=1).head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:04:33.611954Z","iopub.execute_input":"2025-10-16T18:04:33.612242Z","iopub.status.idle":"2025-10-16T18:04:33.659095Z","shell.execute_reply.started":"2025-10-16T18:04:33.612220Z","shell.execute_reply":"2025-10-16T18:04:33.658275Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_rolling_stats(df: pd.DataFrame, windows=[5, 15, 45, 120]):\n    \"\"\"إضافة mean/std/median/percentile لكل عمود عددياً عبر نوافذ مختلفة\"\"\"\n    out = df.copy()\n    for w in windows:\n        suffix = f'_{w}'\n        out = out.assign(**{\n            f'{col}_rmean{suffix}': df[col].rolling(w, min_periods=1, center=True).mean()\n            for col in df.columns\n        })\n        out = out.assign(**{\n            f'{col}_rstd{suffix}': df[col].rolling(w, min_periods=1, center=True).std().fillna(0)\n            for col in df.columns\n        })\n    return out\n\n# تجربة سريعة\nif 'f1' in globals():\n    rolled = add_rolling_stats(f1[['mouse1_speed','mouse1_accel']], windows=[5,15,45])\n    display(rolled.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:18:28.072966Z","iopub.execute_input":"2025-10-16T18:18:28.073246Z","iopub.status.idle":"2025-10-16T18:18:28.130364Z","shell.execute_reply.started":"2025-10-16T18:18:28.073225Z","shell.execute_reply":"2025-10-16T18:18:28.129446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, gc\nimport numpy as np\nimport pandas as pd\nfrom tqdm.auto import tqdm\nfrom scipy.stats import skew, kurtosis\nfrom scipy.fft import rfft\n\n# ✅ استخدام polars لو موجود للتسريع\ntry:\n    import polars as pl\n    USE_POLARS = True\n    print(\"Using polars\")\nexcept:\n    USE_POLARS = False\n    print(\"polars not available — falling back to pandas\")\n\nINPUT_DIR = \"/kaggle/input/MABe-mouse-behavior-detection\"\nprint(\"Input exists?\", os.path.exists(INPUT_DIR))\nprint(\"Top-level files/folders:\")\nfor p in sorted(os.listdir(INPUT_DIR))[:200]:\n    print(\" \", p)\n\n# 🔹 قراءة ملف parquet\ndef read_tracking_parquet(path: str):\n    if USE_POLARS:\n        df = pl.read_parquet(path).to_pandas()\n    else:\n        df = pd.read_parquet(path)\n    print(\"Loaded\", path, \"shape:\", df.shape)\n    print(df.columns.tolist()[:40])\n    return df\n\n# 🔹 إيجاد أول ملف parquet للتجربة\nfound = False\nfor folder in [\"train_tracking\", \"test_tracking\"]:\n    folder_path = os.path.join(INPUT_DIR, folder)\n    if os.path.exists(folder_path):\n        for lab in sorted(os.listdir(folder_path))[:5]:\n            lab_dir = os.path.join(folder_path, lab)\n            if os.path.isdir(lab_dir):\n                files = [f for f in os.listdir(lab_dir) if f.endswith(\".parquet\")]\n                if files:\n                    test_path = os.path.join(lab_dir, files[0])\n                    print(\"Sample parquet:\", test_path)\n                    df_sample = read_tracking_parquet(test_path)\n                    found = True\n                    break\n    if found: break\n\nif not found:\n    print(\"No parquet found in expected folders — check INPUT_DIR contents.\")\n\n# 🔹 Pivot للـ body_center لكل فأر\ndef pivot_tracking(df):\n    frame_col = next((c for c in ['video_frame','frame','frame_id','frameId'] if c in df.columns), None)\n    assert frame_col is not None, \"Cannot find frame column in tracking file\"\n    df = df.rename(columns={frame_col: 'video_frame'})\n    assert 'mouse_id' in df.columns and 'bodypart' in df.columns and 'x' in df.columns and 'y' in df.columns\n    p = df.pivot_table(index='video_frame', columns=['mouse_id','bodypart'], values=['x','y'], aggfunc='first')\n    p = p.sort_index(axis=1, level=[0,1])\n    return p\n\n# 🔹 حساب مراكز الفأر\ndef compute_body_centers(pivot_df):\n    cols = pivot_df.columns\n    mice = sorted(set([c[1] for c in cols]))\n    centers = pd.DataFrame(index=pivot_df.index)\n    for m in mice:\n        x_cols = [c for c in cols if c[1]==m and c[0]=='x']\n        y_cols = [c for c in cols if c[1]==m and c[0]=='y']\n        if not x_cols or not y_cols:\n            continue\n        centers[f'mouse{m}_cx'] = pivot_df[x_cols].mean(axis=1)\n        centers[f'mouse{m}_cy'] = pivot_df[y_cols].mean(axis=1)\n    return centers\n\n# 🔹 Motion features لكل فأر\ndef add_motion_features(centers: pd.DataFrame, prefix: str):\n    if f'{prefix}_cx' not in centers.columns or f'{prefix}_cy' not in centers.columns:\n        return pd.DataFrame(index=centers.index)  # إذا ما موجودش العمود\n    cx = centers[f'{prefix}_cx']\n    cy = centers[f'{prefix}_cy']\n    vx = cx.diff().fillna(0.0)\n    vy = cy.diff().fillna(0.0)\n    speed = np.sqrt(vx**2 + vy**2)\n    ax = vx.diff().fillna(0.0)\n    ay = vy.diff().fillna(0.0)\n    accel = np.sqrt(ax**2 + ay**2)\n    heading = np.arctan2(vy, vx).fillna(0.0)\n    heading_unwrapped = np.unwrap(heading.to_numpy())\n    ang_diff = np.abs(np.diff(heading_unwrapped, prepend=heading_unwrapped[0]))\n    ang_diff = pd.Series(ang_diff, index=centers.index)\n    features = pd.DataFrame({\n        f'{prefix}_vx': vx,\n        f'{prefix}_vy': vy,\n        f'{prefix}_speed': speed,\n        f'{prefix}_ax': ax,\n        f'{prefix}_ay': ay,\n        f'{prefix}_accel': accel,\n        f'{prefix}_heading': heading,\n        f'{prefix}_angdiff': ang_diff\n    }, index=centers.index)\n    return features\n\n# 🔹 Relative / Pair features\ndef add_relative_features(centers: pd.DataFrame):\n    mice = sorted(set([c.split('_')[0] for c in centers.columns]))\n    rel_feats = pd.DataFrame(index=centers.index)\n    for i,a in enumerate(mice):\n        for j,b in enumerate(mice):\n            if i >= j:\n                continue\n            if f'{a}_cx' not in centers.columns or f'{b}_cx' not in centers.columns:\n                continue\n            dx = centers[f'{b}_cx'] - centers[f'{a}_cx']\n            dy = centers[f'{b}_cy'] - centers[f'{a}_cy']\n            dist = np.sqrt(dx**2 + dy**2)\n            angle = np.arctan2(dy, dx)\n            rel_feats[f'{a}_{b}_dist'] = dist\n            rel_feats[f'{a}_{b}_angle'] = angle\n            rel_feats[f'{a}_{b}_approach_rate'] = dist.diff().fillna(0)\n    return rel_feats\n\n# 🔹 Rolling stats متقدمة\ndef add_advanced_rolling_stats(df: pd.DataFrame, windows=[5,15,45,90]):\n    out = df.copy()\n    numeric_cols = df.select_dtypes(include=np.number).columns\n    for w in windows:\n        for col in numeric_cols:\n            roll = df[col].rolling(w, min_periods=1, center=True)\n            out[f'{col}_rmean_{w}'] = roll.mean()\n            out[f'{col}_rstd_{w}'] = roll.std().fillna(0)\n            out[f'{col}_rmedian_{w}'] = roll.median()\n            out[f'{col}_rmin_{w}'] = roll.min()\n            out[f'{col}_rmax_{w}'] = roll.max()\n            out[f'{col}_rskew_{w}'] = roll.apply(lambda x: skew(x), raw=True)\n            out[f'{col}_rkurt_{w}'] = roll.apply(lambda x: kurtosis(x), raw=True)\n    return out\n\n# 🔹 Fourier features\ndef add_frequency_features(centers: pd.DataFrame, n_freq=5):\n    mice = sorted(set([c.split('_')[0] for c in centers.columns]))\n    freq_feats = pd.DataFrame(index=centers.index)\n    for m in mice:\n        if f'{m}_cx' not in centers.columns:\n            continue\n        vx = centers[f'{m}_cx'].diff().fillna(0)\n        vy = centers[f'{m}_cy'].diff().fillna(0)\n        speed = np.sqrt(vx**2 + vy**2)\n        fft_vals = np.abs(rfft(speed))\n        top_n = min(n_freq, len(fft_vals))\n        for k in range(top_n):\n            col_name = f'{m}_fft_{k}'\n            freq_feats[col_name] = fft_vals[k]\n    return freq_feats\n\n# 🔹 استخراج كل features المتقدمة (Top 5)\ndef extract_top5_features(centers: pd.DataFrame):\n    all_feats = []\n    mice = sorted(set([c.split('_')[0] for c in centers.columns]))\n    for m in mice:\n        f = add_motion_features(centers, m)\n        if not f.empty:\n            all_feats.append(f)\n    motion_df = pd.concat(all_feats, axis=1)\n    pair_df = add_relative_features(centers)\n    combined_df = pd.concat([motion_df, pair_df], axis=1)\n    rolled_df = add_advanced_rolling_stats(combined_df, windows=[5,15,45,90])\n    fft_df = add_frequency_features(centers, n_freq=5)\n    final_df = pd.concat([rolled_df, fft_df], axis=1)\n    return final_df\n\n# 🔹 تجربة على أول 500 فريم\nif 'df_sample' in globals():\n    pivoted = pivot_tracking(df_sample)\n    centers = compute_body_centers(pivoted)\n    top5_features = extract_top5_features(centers.iloc[:500])\n    display(top5_features.head())\n    print(\"Shape of final features:\", top5_features.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:46:35.505585Z","iopub.execute_input":"2025-10-16T18:46:35.505868Z","iopub.status.idle":"2025-10-16T18:47:47.863509Z","shell.execute_reply.started":"2025-10-16T18:46:35.505849Z","shell.execute_reply":"2025-10-16T18:47:47.862437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 🔹 Behavioral advanced features\ndef add_behavioral_features(centers: pd.DataFrame, distance_thresh=50, angle_thresh=np.pi/4):\n    mice = sorted(set([c.split('_')[0] for c in centers.columns]))\n    beh_feats = pd.DataFrame(index=centers.index)\n    \n    for m in mice:\n        if f'{m}_cx' not in centers.columns: continue\n        vx = centers[f'{m}_cx'].diff().fillna(0)\n        vy = centers[f'{m}_cy'].diff().fillna(0)\n        speed = np.sqrt(vx**2 + vy**2)\n        accel = np.sqrt(vx.diff().fillna(0)**2 + vy.diff().fillna(0)**2)\n        heading = np.arctan2(vy, vx).fillna(0)\n        heading_unwrapped = np.unwrap(heading.to_numpy())\n        ang_diff = np.abs(np.diff(heading_unwrapped, prepend=heading_unwrapped[0]))\n\n        # 1️⃣ Acceleration peaks (> 95th percentile)\n        acc_peak_thresh = np.percentile(accel, 95)\n        beh_feats[f'{m}_acc_peak'] = (accel > acc_peak_thresh).astype(int)\n\n        # 2️⃣ Direction changes (> angle_thresh)\n        beh_feats[f'{m}_dir_change'] = (ang_diff > angle_thresh).astype(int)\n\n        # 5️⃣ Cumulative distance\n        beh_feats[f'{m}_cum_dist'] = speed.cumsum()\n\n    # 3️⃣ و 4️⃣ Relative features بين كل زوج\n    for i,a in enumerate(mice):\n        for j,b in enumerate(mice):\n            if i >= j: continue\n            if f'{a}_cx' not in centers.columns or f'{b}_cx' not in centers.columns: continue\n            dx = centers[f'{b}_cx'] - centers[f'{a}_cx']\n            dy = centers[f'{b}_cy'] - centers[f'{a}_cy']\n            dist = np.sqrt(dx**2 + dy**2)\n            speed_diff = np.abs(np.sqrt(dx.diff().fillna(0)**2 + dy.diff().fillna(0)**2))\n            \n            beh_feats[f'{a}_{b}_prox_alert'] = (dist < distance_thresh).astype(int)\n            beh_feats[f'{a}_{b}_rel_speed'] = speed_diff\n            \n    return beh_feats\n\n# 🔹 استخراج كل الـ features المتقدمة + behavioral\ndef extract_top5_behavioral_features(centers: pd.DataFrame):\n    all_feats = []\n    mice = sorted(set([c.split('_')[0] for c in centers.columns]))\n    for m in mice:\n        f = add_motion_features(centers, m)\n        if not f.empty: all_feats.append(f)\n    motion_df = pd.concat(all_feats, axis=1)\n    pair_df = add_relative_features(centers)\n    combined_df = pd.concat([motion_df, pair_df], axis=1)\n    rolled_df = add_advanced_rolling_stats(combined_df, windows=[5,15,45,90])\n    fft_df = add_frequency_features(centers, n_freq=5)\n    behavioral_df = add_behavioral_features(centers)\n    \n    final_df = pd.concat([rolled_df, fft_df, behavioral_df], axis=1)\n    return final_df\n\n# 🔹 تجربة على أول 500 فريم\nif 'df_sample' in globals():\n    pivoted = pivot_tracking(df_sample)\n    centers = compute_body_centers(pivoted)\n    top5_behavioral_features = extract_top5_behavioral_features(centers.iloc[:500])\n    display(top5_behavioral_features.head())\n    print(\"Shape of final behavioral features:\", top5_behavioral_features.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T18:48:04.507074Z","iopub.execute_input":"2025-10-16T18:48:04.507404Z","iopub.status.idle":"2025-10-16T18:49:16.632274Z","shell.execute_reply.started":"2025-10-16T18:48:04.507380Z","shell.execute_reply":"2025-10-16T18:49:16.631397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nimport os, gc\nimport glob\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm.auto import tqdm\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import log_loss, roc_auc_score\nimport lightgbm as lgb\nimport joblib\n\n# ----------------- CONFIG -----------------\nINPUT_DIR = \"/kaggle/input/MABe-mouse-behavior-detection\"\nTRAIN_TRACKING = os.path.join(INPUT_DIR, \"train_tracking\")\nTRAIN_ANNOT = os.path.join(INPUT_DIR, \"train_annotation\")\nCACHE_DIR = \"/kaggle/working/features_cache\"\nos.makedirs(CACHE_DIR, exist_ok=True)\n\nN_FOLDS = 5\nRANDOM_STATE = 42\nNUM_BOOST_ROUND = 3000\nEARLY_STOP = 100\nVERBOSE = 100\n\n\nLIMIT_VIDEOS = None     \nLIMIT_FRAMES_PER_VIDEO = None  \n\ndef load_annotations():\n    \"\"\"\n    Try to read annotations in various expected locations.\n    Returns DataFrame with at least: ['video_id','video_frame', <target columns>]\n    \"\"\"\n    # try a top-level train.csv first\n    train_csv = os.path.join(INPUT_DIR, \"train.csv\")\n    if os.path.exists(train_csv):\n        ann = pd.read_csv(train_csv)\n        print(\"Loaded annotations from\", train_csv, \"shape:\", ann.shape)\n        return ann\n\n    if os.path.exists(TRAIN_ANNOT):\n        rows = []\n        for p in sorted(glob.glob(os.path.join(TRAIN_ANNOT, \"**\", \"*.csv\"), recursive=True)):\n            df = pd.read_csv(p)\n            # try to infer video_id column or take parent folder name\n            if 'video_id' not in df.columns and 'video' not in df.columns:\n                # attempt to derive from filename\n                vid = Path(p).stem\n                df['video_id'] = vid\n            rows.append(df)\n        if rows:\n            ann = pd.concat(rows, ignore_index=True)\n            print(\"Loaded annotations from\", TRAIN_ANNOT, \"shape:\", ann.shape)\n            return ann\n\n    raise FileNotFoundError(\"No annotation file found; please place train.csv or check train_annotation folder.\")\n\n\ndef iter_tracking_files(limit=None):\n    paths = []\n    for lab in sorted(os.listdir(TRAIN_TRACKING)):\n        lab_dir = os.path.join(TRAIN_TRACKING, lab)\n        if not os.path.isdir(lab_dir): continue\n        files = sorted([os.path.join(lab_dir, f) for f in os.listdir(lab_dir) if f.endswith(\".parquet\")])\n        paths.extend(files)\n        if limit and len(paths) >= limit:\n            break\n    if limit:\n        paths = paths[:limit]\n    return paths\n\ndef process_one_parquet(path_parquet, force_recompute=False, n_frames_limit=None):\n    \"\"\"\n    Reads a tracking parquet, computes centers and advanced features,\n    returns DataFrame features with columns: video_id, video_frame, <features...>\n    Caches to CACHE_DIR by file stem.\n    \"\"\"\n    stem = Path(path_parquet).stem\n    cache_file = os.path.join(CACHE_DIR, f\"{stem}_features.parquet\")\n    if os.path.exists(cache_file) and not force_recompute:\n        df = pd.read_parquet(cache_file)\n        return df\n\n\n    try:\n        raw = read_tracking_parquet(path_parquet)  \n    except Exception:\n        raw = pd.read_parquet(path_parquet)\n\n\n    if n_frames_limit:\n        raw = raw.iloc[:n_frames_limit]\n\n\n    p = pivot_tracking(raw)  \n    centers = compute_body_centers(p)\n\n    centers.index.name = 'video_frame'\n\n\n    try:\n        feats = extract_top5_behavioral_features(centers) \n    except Exception:\n        feats = extract_top5_features(centers)\n\n\n    video_id = Path(path_parquet).parent.name\n    feats = feats.reset_index().rename(columns={'index': 'video_frame'}) if 'video_frame' not in feats.columns else feats.reset_index()\n    feats['video_id'] = video_id\n\n\n    feats.to_parquet(cache_file, index=False)\n    return feats\n\ndef build_feature_table(limit_videos=None, frames_limit=None, force_recompute=False):\n    tracking_files = iter_tracking_files(limit=limit_videos)\n    dfs = []\n    for p in tqdm(tracking_files, desc=\"Processing tracking files\"):\n        df_feats = process_one_parquet(p, force_recompute=force_recompute, n_frames_limit=frames_limit)\n        dfs.append(df_feats)\n    features_all = pd.concat(dfs, ignore_index=True)\n    print(\"Combined features shape:\", features_all.shape)\n\n\n    ann = load_annotations()\n\n\n    if 'video_frame' not in ann.columns:\n        for alt in ['frame', 'frame_id', 'frameId']:\n            if alt in ann.columns:\n                ann = ann.rename(columns={alt: 'video_frame'})\n                break\n    if 'video_id' not in ann.columns and 'video' in ann.columns:\n        ann = ann.rename(columns={'video': 'video_id'})\n\n\n    merged = features_all.merge(ann, how='left', on=['video_id', 'video_frame'])\n    print(\"After merge shape:\", merged.shape)\n    return merged\n\ndef prepare_training_data(merged_df, exclude_cols=None):\n    if exclude_cols is None:\n        exclude_cols = ['video_id', 'video_frame']\n\n    ann_cols = [c for c in merged_df.columns if c not in merged_df.select_dtypes(include=np.number).columns]\n    \n    ann = load_annotations()\n    target_cols = [c for c in ann.columns if c not in ['video_id', 'video_frame']]\n    if len(target_cols) == 0:\n        \n        for name in ['label', 'behavior', 'behavior_id', 'label_id', 'target']:\n            if name in merged_df.columns:\n                target_cols = [name]\n                break\n\n    if len(target_cols) == 0:\n        raise ValueError(\"Cannot find target columns in annotations. Check annotation files and column names.\")\n\n    feature_cols = [c for c in merged_df.select_dtypes(include=np.number).columns if c not in (exclude_cols + target_cols)]\n    print(\"Detected target columns:\", target_cols)\n    print(\"Feature count:\", len(feature_cols))\n    return merged_df, feature_cols, target_cols\n\ndef train_lgb_groupkfold(df, feature_cols, target_cols, n_folds=N_FOLDS, random_state=RANDOM_STATE):\n\n    if 'video_id' not in df.columns:\n        raise ValueError(\"video_id column is required for GroupKFold\")\n\n    groups = df['video_id'].values\n    gkf = GroupKFold(n_splits=n_folds)\n\n    oof_preds = {}\n    models = []\n    importance_df = pd.DataFrame(index=feature_cols)\n\n\n    for target in target_cols:\n        print(\"Training target:\", target)\n        y = df[target].values\n        oof = np.zeros(len(df))\n        fold_importances = pd.DataFrame()\n        for fold, (train_idx, val_idx) in enumerate(gkf.split(df, y, groups)):\n            print(f\"Fold {fold+1}/{n_folds} — train {len(train_idx)} val {len(val_idx)}\")\n            X_train = df.loc[train_idx, feature_cols]\n            X_val = df.loc[val_idx, feature_cols]\n            y_train = y[train_idx]\n            y_val = y[val_idx]\n\n            lgb_train = lgb.Dataset(X_train, y_train)\n            lgb_val = lgb.Dataset(X_val, y_val, reference=lgb_train)\n\n        \n            unique_vals = np.unique(y_train[~pd.isna(y_train)])\n            if len(unique_vals) <= 2:\n                params = {\n                    'objective': 'binary',\n                    'metric': 'binary_logloss',\n                    'verbosity': -1,\n                    'boosting_type': 'gbdt',\n                    'seed': random_state,\n                    'deterministic': True,\n                }\n            else:\n                params = {\n                    'objective': 'multiclass',\n                    'num_class': int(len(unique_vals)),\n                    'metric': 'multi_logloss',\n                    'verbosity': -1,\n                    'seed': random_state,\n                    'deterministic': True,\n                }\n\n            model = lgb.train(\n                params,\n                lgb_train,\n                valid_sets=[lgb_train, lgb_val],\n                verbose_eval=VERBOSE,\n                num_boost_round=NUM_BOOST_ROUND,\n                early_stopping_rounds=EARLY_STOP\n            )\n\n            models.append((target, fold, model))\n        \n            if params['objective'] == 'binary':\n                preds = model.predict(X_val, num_iteration=model.best_iteration)\n                oof[val_idx] = preds\n            else:\n                preds = model.predict(X_val, num_iteration=model.best_iteration)\n                \n                oof[val_idx] = np.argmax(preds, axis=1)\n\n            # feature importance\n            fi = pd.DataFrame()\n            fi['feature'] = feature_cols\n            fi['importance'] = model.feature_importance(importance_type='gain')\n            fi['fold'] = fold\n            fi['target'] = target\n            fold_importances = pd.concat([fold_importances, fi], ignore_index=True)\n\n            # cleanup\n            del X_train, X_val, lgb_train, lgb_val\n            gc.collect()\n\n        oof_preds[target] = oof\n        importance_df = importance_df.join(\n            fold_importances.groupby('feature')['importance'].mean().rename(f'{target}_importance'),\n            how='left'\n        )\n\n    return oof_preds, models, importance_df\n\nmerged = build_feature_table(limit_videos=LIMIT_VIDEOS, frames_limit=LIMIT_FRAMES_PER_VIDEO, force_recompute=False)\n\n\nmerged, feature_cols, target_cols = prepare_training_data(merged)\n\n\noof_preds, models, importance_df = train_lgb_groupkfold(merged, feature_cols, target_cols, n_folds=N_FOLDS)\n\n\nimportance_df_mean = importance_df.mean(axis=1).sort_values(ascending=False)\nprint(\"Top 50 features by average importance:\")\ndisplay(importance_df_mean.head(50))\n\n\njoblib.dump(models, \"/kaggle/working/lgb_models.pkl\")\nfor t, preds in oof_preds.items():\n    pd.DataFrame({f\"{t}_oof\": preds}).to_csv(f\"/kaggle/working/oof_{t}.csv\", index=False)\n\nprint(\"Training complete. Models saved to /kaggle/working/lgb_models.pkl\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-16T19:00:30.997629Z","iopub.execute_input":"2025-10-16T19:00:30.997964Z","execution_failed":"2025-10-17T06:12:58.706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nsubmission_rows = []\n\nthreshold = 0.5\n\nfor target, preds in oof_preds.items():\n    df_pred = merged[['video_id', 'video_frame']].copy()\n    df_pred['prob'] = preds\n    df_pred['target'] = target\n\n    for vid in df_pred['video_id'].unique():\n        df_vid = df_pred[df_pred['video_id'] == vid].sort_values('video_frame')\n        frames = df_vid['video_frame'].values\n        probs = df_vid['prob'].values\n\n        active = probs > threshold\n        start_frame = None\n\n        for i, is_active in enumerate(active):\n            if is_active and start_frame is None:\n                start_frame = frames[i]\n            elif not is_active and start_frame is not None:\n                stop_frame = frames[i-1]\n                submission_rows.append([vid, target, target, target, start_frame, stop_frame])\n                start_frame = None\n\n        if start_frame is not None:\n            submission_rows.append([vid, target, target, target, start_frame, frames[-1]])\n\nsubmission_df = pd.DataFrame(submission_rows, columns=['video_id','agent_id','target_id','action','start_frame','stop_frame'])\nsubmission_df.insert(0, 'row_id', np.arange(len(submission_df)))\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\n\nprint(\"Professional submission file created at /kaggle/working/submission.csv\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"output.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}