{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13719397,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\ncount = 0\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        count += 1\n        if count >= 20:  # stop after printing 20 files\n            break\n    if count >= 20:\n        break\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-27T17:24:28.050530Z","iopub.execute_input":"2025-09-27T17:24:28.050702Z","iopub.status.idle":"2025-09-27T17:24:30.804744Z","shell.execute_reply.started":"2025-09-27T17:24:28.050686Z","shell.execute_reply":"2025-09-27T17:24:30.803889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Kaggle/Colab friendly setup (CPU by default; optional CuPy if available)\nimport os, sys, math, json, gc, warnings, random, itertools\nfrom collections import defaultdict, Counter\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\")\n\ndef find_kaggle_root() -> str:\n    base = \"/kaggle/input\"\n    if not os.path.exists(base):\n        return os.getcwd()\n    # Prefer dirs containing test.csv\n    for item in os.listdir(base):\n        p = os.path.join(base, item)\n        if os.path.isdir(p) and os.path.exists(os.path.join(p, \"test.csv\")):\n            return p\n        if os.path.isdir(p):\n            for sub in os.listdir(p):\n                sp = os.path.join(p, sub)\n                if os.path.isdir(sp) and os.path.exists(os.path.join(sp, \"test.csv\")):\n                    return sp\n    # Fallback\n    for item in os.listdir(base):\n        p = os.path.join(base, item)\n        if os.path.isdir(p):\n            return p\n    return base\n\nROOT = find_kaggle_root()\nWORK = \"/kaggle/working\" if os.path.exists(\"/kaggle/working\") else os.getcwd()\nprint(\"ROOT:\", ROOT)\nprint(\"WORK:\", WORK)\n\n# Optional GPU accel for distance via CuPy (set True to try; will fallback)\nUSE_GPU = True\ntry:\n    import cupy as cp  # type: ignore\n    HAS_CUPY = True\n    print('Gpu In use')\nexcept Exception:\n    HAS_CUPY = False\n    USE_GPU = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:37.960882Z","iopub.execute_input":"2025-09-24T10:55:37.961133Z","iopub.status.idle":"2025-09-24T10:55:37.969937Z","shell.execute_reply.started":"2025-09-24T10:55:37.961116Z","shell.execute_reply":"2025-09-24T10:55:37.969123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def safe_le(a, b, valid):\n    a = np.asarray(a)\n    out = np.zeros(a.shape, dtype=bool)\n    finite = np.isfinite(a)\n    np.less_equal(a, b, out=out, where=(valid & finite))\n    return out\n\ndef safe_ge(a, b, valid):\n    a = np.asarray(a)\n    out = np.zeros(a.shape, dtype=bool)\n    finite = np.isfinite(a)\n    np.greater_equal(a, b, out=out, where=(valid & finite))\n    return out\n\ndef nan_safe_hypot(dx: np.ndarray, dy: np.ndarray) -> np.ndarray:\n    valid = np.isfinite(dx) & np.isfinite(dy)\n    out = np.full(dx.shape, np.nan, dtype=float)\n    if USE_GPU and HAS_CUPY:\n        try:\n            dxg, dyg, valg = cp.asarray(dx), cp.asarray(dy), cp.asarray(valid)\n            outg = cp.full(dxg.shape, cp.nan, dtype=cp.float64)\n            cp.hypot(dxg, dyg, out=outg, where=valg)\n            return cp.asnumpy(outg)\n        except Exception:\n            pass\n    np.hypot(dx, dy, out=out, where=valid)\n    return out\n\ndef read_metadata(path_csv: str) -> pd.DataFrame:\n    if not os.path.exists(path_csv):\n        raise FileNotFoundError(path_csv)\n    df = pd.read_csv(path_csv)\n    rename = {\n        \"frames per second\": \"fps\",\n        \"frames_per_second\": \"fps\",\n        \"pix per cm (approx)\": \"pix_per_cm\",\n        \"pix_per_cm_approx\": \"pix_per_cm\",\n    }\n    for k, v in rename.items():\n        if k in df.columns: df.rename(columns={k: v}, inplace=True)\n    if \"fps\" not in df.columns:\n        for alt in [\"frame_rate\", \"Frames per second\"]:\n            if alt in df.columns: df.rename(columns={alt: \"fps\"}, inplace=True)\n    if \"pix_per_cm\" not in df.columns:\n        for alt in [\"Pixels per cm\", \"pixels_per_cm\"]:\n            if alt in df.columns: df.rename(columns={alt: \"pix_per_cm\"}, inplace=True)\n    return df\n\ndef load_tracking_parquet(root: str, split: str, lab_id: str, video_id: int) -> pd.DataFrame:\n    p = os.path.join(root, f\"{split}_tracking\", str(lab_id), f\"{int(video_id)}.parquet\")\n    if not os.path.exists(p):\n        raise FileNotFoundError(p)\n    df = pd.read_parquet(p)\n    # Standardize\n    ren = {}\n    if \"video_frame\" not in df.columns:\n        for alt in [\"frame\", \"frame_idx\", \"frame_index\", \"frame_id\"]:\n            if alt in df.columns: ren[alt] = \"video_frame\"; break\n    if \"mouse_id\" not in df.columns:\n        for alt in [\"individual\", \"track_id\", \"id\", \"agent_id\"]:\n            if alt in df.columns: ren[alt] = \"mouse_id\"; break\n    if \"x\" not in df.columns:\n        for alt in [\"x_px\", \"xpos\", \"x_pos\", \"X\", \"x_coord\"]:\n            if alt in df.columns: ren[alt] = \"x\"; break\n    if \"y\" not in df.columns:\n        for alt in [\"y_px\", \"ypos\", \"y_pos\", \"Y\", \"y_coord\"]:\n            if alt in df.columns: ren[alt] = \"y\"; break\n    if ren: df = df.rename(columns=ren)\n    if \"bodypart\" not in df.columns:\n        for alt in [\"body_part\", \"node\", \"keypoint\", \"part\", \"BodyPart\"]:\n            if alt in df.columns: df = df.rename(columns={alt: \"bodypart\"}); break\n    for c in [\"video_frame\",\"mouse_id\",\"x\",\"y\"]:\n        if c not in df.columns: raise ValueError(f\"Missing {c} in {p}\")\n    return df\n\ndef bodypart_mask(series: pd.Series, keywords) -> pd.Series:\n    lo = series.astype(str).str.lower()\n    m = pd.Series(False, index=series.index)\n    for kw in keywords:\n        m = m | lo.str.contains(kw, na=False)\n    return m","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:37.970864Z","iopub.execute_input":"2025-09-24T10:55:37.971099Z","iopub.status.idle":"2025-09-24T10:55:37.986710Z","shell.execute_reply.started":"2025-09-24T10:55:37.971083Z","shell.execute_reply":"2025-09-24T10:55:37.985898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_centroids(trk: pd.DataFrame) -> pd.DataFrame:\n    cent = (trk.groupby([\"video_frame\",\"mouse_id\"], as_index=False)[[\"x\",\"y\"]]\n              .mean().rename(columns={\"x\":\"centroid_x\",\"y\":\"centroid_y\"}))\n    parts = {\n        \"nose\": [\"nose\",\"snout\",\"sniff\",\"snout_tip\"],\n        \"head\": [\"head\",\"ear\",\"neck\",\"head_center\"],\n        \"rear\": [\"rear\",\"back\",\"rump\",\"hindquarters\"],\n        \"tail\": [\"tail\",\"tail_base\",\"tail_tip\",\"tail_midpoint\"],\n        \"body\": [\"body\",\"center\",\"lateral\",\"body_center\"],\n    }\n    if \"bodypart\" in trk.columns:\n        for name, kws in parts.items():\n            m = bodypart_mask(trk[\"bodypart\"], kws)\n            if m.any():\n                sub = (trk.loc[m].groupby([\"video_frame\",\"mouse_id\"], as_index=False)[[\"x\",\"y\"]]\n                         .mean().rename(columns={\"x\":f\"{name}_x\",\"y\":f\"{name}_y\"}))\n                cent = cent.merge(sub, on=[\"video_frame\",\"mouse_id\"], how=\"left\")\n    return cent\n\ndef add_movement(centroids: pd.DataFrame, fps: float, pix_per_cm: float|None) -> pd.DataFrame:\n    if not fps or fps<=0 or pix_per_cm is None or pix_per_cm<=0:\n        for c in [\"speed_cm_s\",\"accel_cm_s2\",\"angular_velocity\",\"curvature\",\"direction_change\"]:\n            centroids[c] = np.nan\n        return centroids\n    df = centroids.sort_values([\"mouse_id\",\"video_frame\"]).copy()\n    for mid, sub in df.groupby(\"mouse_id\"):\n        x, y = sub[\"centroid_x\"].values, sub[\"centroid_y\"].values\n        frames = sub[\"video_frame\"].values\n        dx, dy = np.diff(x), np.diff(y)\n        dt = np.diff(frames)/fps\n        dt = np.where(dt==0, 1.0/fps, dt)\n        vx, vy = (dx/pix_per_cm)/dt, (dy/pix_per_cm)/dt\n        speed = np.sqrt(vx**2 + vy**2)\n        if len(speed)>1:\n            speed = np.convolve(speed, np.ones(2)/2.0, mode=\"same\")\n        accel = np.diff(speed)*fps\n        angles = np.arctan2(vy, vx)\n        ang_vel = np.diff(angles)*fps\n        curv = np.abs(np.diff(ang_vel))*fps\n        dirchg = np.abs(np.diff(angles)); dirchg = np.minimum(dirchg, 2*np.pi - dirchg)\n\n        # pad\n        speed_p = np.concatenate([[0], speed])\n        accel_p = np.concatenate([[0,0], accel])\n        ang_p   = np.concatenate([[0], ang_vel])\n        curv_p  = np.concatenate([[0,0], curv])\n        dir_p   = np.concatenate([[0], dirchg])\n\n        n = len(sub)\n        cols = {\n            \"speed_cm_s\": speed_p[:n],\n            \"accel_cm_s2\": accel_p[:n],\n            \"angular_velocity\": ang_p[:n],\n            \"curvature\": curv_p[:n],\n            \"direction_change\": dir_p[:n],\n        }\n        for k,v in cols.items():\n            df.loc[sub.index, k] = v\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:37.987602Z","iopub.execute_input":"2025-09-24T10:55:37.987813Z","iopub.status.idle":"2025-09-24T10:55:38.000624Z","shell.execute_reply.started":"2025-09-24T10:55:37.987798Z","shell.execute_reply":"2025-09-24T10:55:37.999965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quick EDA\nmeta_train = read_metadata(os.path.join(ROOT, \"train.csv\"))\nmeta_test  = read_metadata(os.path.join(ROOT, \"test.csv\"))\n\nplt.figure(figsize=(10,4))\nmeta_train[\"lab_id\"].value_counts().head(20).plot(kind=\"bar\")\nplt.title(\"Train videos by lab_id\")\nplt.tight_layout()\nplt.show()\n\ncols = [c for c in [\"fps\",\"video_duration_sec\",\"pix_per_cm\"] if c in meta_train.columns]\ndisplay(meta_train[cols].describe(include=\"all\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:38.002367Z","iopub.execute_input":"2025-09-24T10:55:38.002956Z","iopub.status.idle":"2025-09-24T10:55:38.324103Z","shell.execute_reply.started":"2025-09-24T10:55:38.002932Z","shell.execute_reply":"2025-09-24T10:55:38.323409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Per-lab distribution (train vs test)\nfig, axs = plt.subplots(1, 2, figsize=(12, 4))\nmeta_train[\"lab_id\"].value_counts().head(15).plot(kind=\"bar\", ax=axs[0], color=\"tab:blue\", title=\"Train videos by lab (Top 15)\")\nmeta_test[\"lab_id\"].value_counts().head(15).plot(kind=\"bar\", ax=axs[1], color=\"tab:green\", title=\"Test videos by lab (Top 15)\")\nplt.tight_layout(); plt.show()\n\n# fps and pix_per_cm histograms\nfig, axs = plt.subplots(1, 2, figsize=(12, 4))\nif \"fps\" in meta_train.columns:\n    meta_train[\"fps\"].dropna().plot(kind=\"hist\", bins=30, ax=axs[0], color=\"tab:purple\", alpha=0.8, title=\"Train FPS distribution\")\nif \"pix_per_cm\" in meta_train.columns:\n    meta_train[\"pix_per_cm\"].dropna().plot(kind=\"hist\", bins=30, ax=axs[1], color=\"tab:orange\", alpha=0.8, title=\"Train pix_per_cm distribution\")\nplt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:38.324716Z","iopub.execute_input":"2025-09-24T10:55:38.324913Z","iopub.status.idle":"2025-09-24T10:55:39.042039Z","shell.execute_reply.started":"2025-09-24T10:55:38.324896Z","shell.execute_reply":"2025-09-24T10:55:39.041330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\nimport plotly.graph_objects as go\nimport ipywidgets as widgets\nfrom ipywidgets import VBox, HBox, Layout\nfrom IPython.display import display, clear_output\n\n# Utility: top-N labs helper\ndef top_n_labs(df, n=15):\n    vc = df[\"lab_id\"].astype(str).value_counts().head(n)\n    return df[df[\"lab_id\"].astype(str).isin(vc.index)].copy()\n\n# 1) Interactive bar of videos by lab (train/test toggle)\ndataset_toggle = widgets.ToggleButtons(options=[\"train\",\"test\"], value=\"train\", description=\"Split:\")\ntopn_slider = widgets.IntSlider(value=15, min=5, max=50, step=1, description=\"Top labs:\")\n\ndef plot_lab_counts(split=\"train\", topn=15):\n    meta = meta_train if split==\"train\" else meta_test\n    df = top_n_labs(meta, topn)\n    vc = df[\"lab_id\"].astype(str).value_counts().reset_index()\n    # In recent pandas, columns are ['lab_id','count']\n    vc.columns = [\"lab_id\", \"count\"]  # ensure consistent names\n    fig = px.bar(vc, x=\"lab_id\", y=\"count\",\n                 labels={\"lab_id\":\"lab_id\",\"count\":\"# videos\"},\n                 title=f\"Videos by lab_id ({split}, top {topn})\")\n    fig.update_layout(xaxis_tickangle=45, height=450)\n    fig.show()\n\n# 2) Interactive scatter: fps vs pix_per_cm colored by lab\ndef scatter_fps_ppcm(split=\"train\"):\n    meta = meta_train if split==\"train\" else meta_test\n    if \"pix_per_cm\" not in meta.columns:\n        print(\"pix_per_cm not available in metadata.\")\n        return\n    df = meta.copy()\n    df[\"lab_id\"] = df[\"lab_id\"].astype(str)\n    fig = px.scatter(df, x=\"fps\", y=\"pix_per_cm\", color=\"lab_id\",\n                     hover_data=[\"video_id\"], title=f\"fps vs pix_per_cm ({split})\")\n    fig.update_traces(marker=dict(size=7, opacity=0.7))\n    fig.update_layout(height=500, legend_itemsizing=\"trace\")\n    fig.show()\n\nwidgets.interact(scatter_fps_ppcm, split=dataset_toggle);\n\n# 3) Interactive distribution of video_duration_sec (if present)\ndef dist_duration(split=\"train\"):\n    meta = meta_train if split==\"train\" else meta_test\n    if \"video_duration_sec\" not in meta.columns:\n        print(\"video_duration_sec not in metadata.\")\n        return\n    fig = px.histogram(meta, x=\"video_duration_sec\", nbins=40,\n                       title=f\"Video duration distribution (sec) - {split}\")\n    fig.update_layout(height=400)\n    fig.show()\n\nwidgets.interact(dist_duration, split=dataset_toggle);","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T10:55:59.228716Z","iopub.execute_input":"2025-09-24T10:55:59.229030Z","iopub.status.idle":"2025-09-24T10:55:59.404510Z","shell.execute_reply.started":"2025-09-24T10:55:59.229009Z","shell.execute_reply":"2025-09-24T10:55:59.403683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef add_movement(centroids: pd.DataFrame, fps: float, pix_per_cm: float | None) -> pd.DataFrame:\n    \"\"\"\n    Adds per-frame kinematics to a centroids dataframe without length mismatches.\n    Expects columns:\n      - 'frame'\n      - 'agent_id' (string or int)\n      - position columns: either ('x','y') or ('cx','cy')\n    Produces columns:\n      - 'speed_cm_s', 'accel_cm_s2', 'angular_velocity', 'curvature', 'direction_change'\n    All derived arrays are computed with np.gradient to keep same length as the index.\n    \"\"\"\n    if centroids is None or len(centroids) == 0:\n        return centroids\n\n    df = centroids.copy()\n\n    # Choose coordinate columns\n    if {'x', 'y'}.issubset(df.columns):\n        x_col, y_col = 'x', 'y'\n    elif {'cx', 'cy'}.issubset(df.columns):\n        x_col, y_col = 'cx', 'cy'\n    else:\n        raise ValueError(\"add_movement: expected position columns ('x','y') or ('cx','cy').\")\n\n    # Ensure dtypes\n    df['frame'] = df['frame'].astype(int)\n    # Keep agent_id as-is (string or int), just ensure existence\n    if 'agent_id' not in df.columns:\n        raise ValueError(\"add_movement: expected 'agent_id' column.\")\n\n    # Prepare output columns\n    out_cols = [\"speed_cm_s\", \"accel_cm_s2\", \"angular_velocity\", \"curvature\", \"direction_change\"]\n    for c in out_cols:\n        if c not in df.columns:\n            df[c] = np.nan\n\n    eps = 1e-8\n    px_to_cm = (1.0 / float(pix_per_cm)) if (pix_per_cm is not None and pix_per_cm > 0) else None\n\n    # Compute per-agent to avoid mixing tracks\n    for _, sub_idx in df.groupby('agent_id').groups.items():\n        sub = df.loc[sub_idx].sort_values('frame')\n        idx = sub.index\n\n        x = sub[x_col].to_numpy(dtype=float)\n        y = sub[y_col].to_numpy(dtype=float)\n\n        # Time derivatives via gradient keep the same length as input\n        # First spatial derivatives (per frame)\n        dx = np.gradient(x)\n        dy = np.gradient(y)\n\n        # Distance per frame (in pixels)\n        ds_pix = np.hypot(dx, dy)\n\n        # Speed: pixels per second -> convert to cm/s if pix_per_cm provided\n        speed_pix_s = ds_pix * float(fps)\n        if px_to_cm is not None:\n            speed_cm_s = speed_pix_s * px_to_cm\n        else:\n            # If pixel scale unknown, still provide speed in \"pixels per second\"\n            speed_cm_s = speed_pix_s\n\n        # Acceleration: time derivative of speed\n        accel_pix_s2 = np.gradient(speed_pix_s) * float(fps)\n        if px_to_cm is not None:\n            accel_cm_s2 = accel_pix_s2 * px_to_cm\n        else:\n            accel_cm_s2 = accel_pix_s2\n\n        # Heading angle and angular velocity\n        theta = np.arctan2(dy, dx)  # radians\n        dtheta = np.gradient(theta)  # radians per frame (same length)\n        # Wrap small numerical jumps\n        dtheta_wrapped = np.arctan2(np.sin(dtheta), np.cos(dtheta))\n        angular_velocity = dtheta_wrapped * float(fps)  # radians per second\n\n        # Curvature kappa ≈ |dtheta/ds|\n        curvature = np.abs(dtheta_wrapped) / (ds_pix + eps)\n\n        # Direction change (per frame, absolute angle change in radians)\n        direction_change = np.abs(dtheta_wrapped)\n\n        # Assign back (lengths match sub.index exactly)\n        df.loc[idx, \"speed_cm_s\"] = speed_cm_s\n        df.loc[idx, \"accel_cm_s2\"] = accel_cm_s2\n        df.loc[idx, \"angular_velocity\"] = angular_velocity\n        df.loc[idx, \"curvature\"] = curvature\n        df.loc[idx, \"direction_change\"] = direction_change\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T11:11:44.797587Z","iopub.execute_input":"2025-09-24T11:11:44.798262Z","iopub.status.idle":"2025-09-24T11:11:44.809106Z","shell.execute_reply.started":"2025-09-24T11:11:44.798237Z","shell.execute_reply":"2025-09-24T11:11:44.808176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation\nfrom IPython.display import Image, display\n\n# ---------------------------\n# Helper Functions\n# ---------------------------\n\ndef quick_explore_parquet(file_path, max_rows=1000):\n    \"\"\"Quick exploration of parquet file structure\"\"\"\n    try:\n        df = pd.read_parquet(file_path).head(max_rows)\n        print(f\"File: {os.path.basename(file_path)}\")\n        print(f\"Shape (first {max_rows} rows): {df.shape}\")\n        print(f\"Columns: {list(df.columns)}\")\n        print(f\"Data types:\\n{df.dtypes}\")\n        print(f\"Sample data:\\n{df.head(3)}\")\n        print(\"-\" * 40)\n        return df\n    except Exception as e:\n        print(f\"Error reading {file_path}: {e}\")\n        return None\n\ndef animate_mouse_bodyparts(df, mouse_id, output_dir=\"animations\", max_frames=500):\n    \"\"\"Create a body-part movement animation for a given mouse\"\"\"\n    \n    mouse_data = df[df['mouse_id'] == mouse_id].copy()  # copy to avoid SettingWithCopyWarning\n\n    # Clean data\n    mouse_data['x'] = mouse_data['x'].replace([np.inf, -np.inf], np.nan)\n    mouse_data['y'] = mouse_data['y'].replace([np.inf, -np.inf], np.nan)\n    mouse_data.dropna(subset=['x', 'y'], inplace=True)\n\n    # Get coordinate ranges\n    x_min, x_max = mouse_data['x'].min(), mouse_data['x'].max()\n    y_min, y_max = mouse_data['y'].min(), mouse_data['y'].max()\n\n    # Unique body parts\n    bodyparts = mouse_data['bodypart'].unique()\n    colors = ['red', 'blue', 'green', 'orange', 'purple', 'brown', 'pink', 'gray', 'olive', 'cyan']\n    \n    if len(bodyparts) > len(colors):\n        import matplotlib.colors as mcolors\n        colors = list(mcolors.TABLEAU_COLORS.values()) + list(mcolors.CSS4_COLORS.values())[:len(bodyparts)-len(colors)]\n\n    # Create figure\n    fig, ax = plt.subplots(figsize=(12, 10))\n    ax.set_xlim(x_min - 10, x_max + 10)\n    ax.set_ylim(y_min - 10, y_max + 10)\n    ax.set_xlabel('X coordinate')\n    ax.set_ylabel('Y coordinate')\n    ax.set_title(f'Mouse {mouse_id} Body Parts Movement Animation')\n\n    # Trajectory lines and points\n    lines, points = [], []\n    for i, bodypart in enumerate(bodyparts):\n        color = colors[i % len(colors)]\n        line, = ax.plot([], [], color=color, alpha=0.5, linewidth=1, label=f'{bodypart}')\n        point, = ax.plot([], [], 'o', color=color, markersize=8, markeredgecolor='black', markeredgewidth=0.5)\n        lines.append(line)\n        points.append(point)\n\n    frame_text = ax.text(0.02, 0.95, '', transform=ax.transAxes, fontsize=12, bbox=dict(facecolor='white', alpha=0.7))\n    ax.legend(loc='upper right')\n\n    frames = mouse_data['video_frame'].unique()\n    frames.sort()\n    \n    if len(frames) > max_frames:\n        step = max(1, len(frames) // max_frames)\n        frames = frames[::step]\n\n    # Animation functions\n    def init():\n        for line, point in zip(lines, points):\n            line.set_data([], [])\n            point.set_data([], [])\n        frame_text.set_text('')\n        return lines + points + [frame_text]\n\n    def update(frame):\n        for i, bodypart in enumerate(bodyparts):\n            bp_data = mouse_data[mouse_data['bodypart'] == bodypart]\n            data = bp_data[bp_data['video_frame'] <= frame]\n            if len(data) > 0:\n                lines[i].set_data(data['x'], data['y'])\n                current = data.iloc[-1]\n                points[i].set_data([current['x']], [current['y']])\n            else:\n                lines[i].set_data([], [])\n                points[i].set_data([], [])\n        frame_text.set_text(f'Frame: {frame}')\n        return lines + points + [frame_text]\n\n    # Create animation\n    ani = FuncAnimation(fig, update, frames=frames, init_func=init, blit=True, interval=100)\n\n    # Save GIF\n    if not os.path.exists(output_dir):\n        os.makedirs(output_dir)\n    output_path = os.path.join(output_dir, f\"mouse{mouse_id}_bodyparts.gif\")\n    ani.save(output_path, writer='pillow', fps=10, dpi=100)\n    plt.close()\n    print(\"Animation GIF saved:\", output_path)\n\n    # Display\n    display(Image(open(output_path, 'rb').read()))\n\n# ---------------------------\n# Main Function\n# ---------------------------\ndef main():\n    train_tracking_path = \"/kaggle/input/MABe-mouse-behavior-detection/train_tracking\"\n\n    if not os.path.exists(train_tracking_path):\n        print(f\"Path not found: {train_tracking_path}\")\n        return\n\n    # Only process first few subdirectories to save memory\n    subdirs = [d for d in os.listdir(train_tracking_path) if os.path.isdir(os.path.join(train_tracking_path, d))][:3]\n    print(f\"Processing {len(subdirs)} subdirectories: {subdirs}\")\n\n    for subdir in subdirs:\n        subdir_path = os.path.join(train_tracking_path, subdir)\n        files = [f for f in os.listdir(subdir_path) if f.endswith('.parquet')]\n        if not files:\n            continue\n        first_file = files[0]\n        file_path = os.path.join(subdir_path, first_file)\n\n        print(f\"\\nProcessing file: {file_path}\")\n        df = quick_explore_parquet(file_path)\n        if df is None:\n            continue\n\n        # Animate first 2 mice only\n        mouse_ids = df['mouse_id'].unique()[:2]\n        for mid in mouse_ids:\n            print(f\"Animating mouse {mid}...\")\n            animate_mouse_bodyparts(df, mid)\n        \n        # Stop after first subdirectory to avoid memory issues\n        break\n\n# ---------------------------\nif __name__ == \"__main__\":\n    main()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-24T11:36:18.666343Z","iopub.execute_input":"2025-09-24T11:36:18.667211Z","iopub.status.idle":"2025-09-24T11:36:39.617759Z","shell.execute_reply.started":"2025-09-24T11:36:18.667182Z","shell.execute_reply":"2025-09-24T11:36:39.616837Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom dataclasses import dataclass\nfrom typing import List, Optional, Dict, Tuple\nfrom tqdm.auto import tqdm\nimport matplotlib.pyplot as plt\nfrom matplotlib.animation import FuncAnimation, PillowWriter\nfrom scipy.spatial.distance import euclidean\nfrom scipy import signal\nfrom sklearn.preprocessing import StandardScaler\n\n# -------------------- Enhanced Parameters --------------------\n@dataclass\nclass EnhancedParams:\n    # Distance thresholds (cm) - More refined based on mouse behavior research\n    sniff_distance_cm: float = 1.5\n    approach_distance_cm: float = 8.0\n    attack_distance_cm: float = 1.2\n    mount_distance_cm: float = 0.8\n    chase_distance_cm: float = 15.0\n    avoid_distance_cm: float = 5.0\n    submit_distance_cm: float = 1.0\n    chaseattack_distance_cm: float = 2.0\n    reciprocalsniff_distance_cm: float = 2.0\n    sniffbody_distance_cm: float = 2.5\n    sniffface_distance_cm: float = 1.8\n    sniffgenital_distance_cm: float = 1.5\n    allogroom_distance_cm: float = 2.0\n    dominancegroom_distance_cm: float = 1.5\n    genitalgroom_distance_cm: float = 1.0\n    huddle_distance_cm: float = 3.0\n    shepherd_distance_cm: float = 6.0\n    follow_distance_cm: float = 8.0\n    exploreobject_distance_cm: float = 4.0\n    biteobject_distance_cm: float = 2.0\n    tussle_distance_cm: float = 2.5\n    disengage_distance_cm: float = 4.0\n    defend_distance_cm: float = 3.0\n    freeze_distance_cm: float = 1.0\n    flinch_distance_cm: float = 2.0\n    \n    # Duration thresholds (seconds) - More realistic minimum durations\n    min_duration_seconds: float = 0.1\n    approach_min_duration: float = 0.3\n    chase_min_duration: float = 0.2\n    attack_min_duration: float = 0.1\n    mount_min_duration: float = 0.5\n    escape_min_duration: float = 0.2\n    rest_min_duration: float = 1.0\n    rear_min_duration: float = 0.3\n    dig_min_duration: float = 0.5\n    selfgroom_min_duration: float = 1.0\n    allogroom_min_duration: float = 0.8\n    climb_min_duration: float = 0.3\n    run_min_duration: float = 0.2\n    freeze_min_duration: float = 0.5\n    sniff_min_duration: float = 0.3\n    huddle_min_duration: float = 2.0\n    \n    # Speed thresholds (cm/s) - Based on mouse locomotion research\n    chase_speed_cm_s: float = 8.0\n    escape_speed_cm_s: float = 12.0\n    rest_speed_cm_s: float = 1.0\n    run_speed_cm_s: float = 5.0\n    follow_speed_cm_s: float = 3.0\n    shepherd_speed_cm_s: float = 6.0\n    freeze_max_speed_cm_s: float = 0.8\n    approach_max_speed_cm_s: float = 4.0\n    \n    # Advanced behavioral parameters\n    velocity_smoothing_window: int = 5\n    acceleration_threshold: float = 15.0\n    direction_change_threshold: float = 45.0  # degrees\n    proximity_history_frames: int = 10\n    behavior_confidence_threshold: float = 0.6\n    \n    # Interaction angle thresholds (degrees)\n    face_angle_threshold: float = 45.0\n    rear_angle_threshold: float = 135.0\n    side_angle_threshold: float = 90.0\n\n# -------------------- Enhanced Data Processing --------------------\ndef _standardize_tracking_columns(df: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Enhanced column standardization with better error handling\"\"\"\n    rename_map = {}\n    col_map = {\n        \"video_frame\": [\"frame\", \"frame_idx\", \"frame_index\", \"frame_id\", \"frameIndex\"],\n        \"mouse_id\": [\"individual\", \"track_id\", \"id\", \"agent_id\", \"mouse\", \"animal_id\"],\n        \"x\": [\"x_px\", \"xpos\", \"x_pos\", \"X\", \"x_coord\", \"pos_x\"],\n        \"y\": [\"y_px\", \"ypos\", \"y_pos\", \"Y\", \"y_coord\", \"pos_y\"]\n    }\n    \n    for std, alts in col_map.items():\n        if std not in df.columns:\n            for alt in alts:\n                if alt in df.columns:\n                    rename_map[alt] = std\n                    break\n    \n    if rename_map: \n        df = df.rename(columns=rename_map)\n    \n    # Ensure required columns exist\n    required = [\"video_frame\", \"mouse_id\", \"x\", \"y\"]\n    missing = [c for c in required if c not in df.columns]\n    if missing:\n        print(f\"Warning: Missing columns {missing}, attempting to infer...\")\n        # Try to infer missing columns from available data\n        if \"mouse_id\" not in df.columns and \"individual\" not in df.columns:\n            # Create dummy mouse IDs if none exist\n            df[\"mouse_id\"] = 0\n    \n    return df\n\ndef enhanced_movement_features(centroids: pd.DataFrame, fps: float, pix_per_cm: Optional[float], params: EnhancedParams) -> pd.DataFrame:\n    \"\"\"Enhanced movement feature computation with better smoothing and error handling\"\"\"\n    df = centroids.sort_values([\"mouse_id\", \"video_frame\"]).copy()\n    \n    # Initialize all feature columns\n    feature_cols = [\"speed_cm_s\", \"accel_cm_s2\", \"direction_deg\", \"angular_velocity\", \n                   \"distance_to_center\", \"smoothed_x\", \"smoothed_y\"]\n    for col in feature_cols:\n        df[col] = np.nan\n    \n    conversion_factor = 1.0 / pix_per_cm if pix_per_cm and pix_per_cm > 0 else 1.0\n    \n    for mouse_id, group in df.groupby(\"mouse_id\"):\n        if len(group) < 3:  # Need at least 3 points for meaningful features\n            continue\n            \n        group = group.sort_values(\"video_frame\")\n        indices = group.index\n        \n        # Get coordinates\n        x_raw = group[\"centroid_x\"].values\n        y_raw = group[\"centroid_y\"].values\n        \n        # Handle missing data with interpolation\n        valid_mask = np.isfinite(x_raw) & np.isfinite(y_raw)\n        if np.sum(valid_mask) < 2:\n            continue\n            \n        # Smooth coordinates\n        window_size = min(params.velocity_smoothing_window, len(x_raw) // 2)\n        if window_size >= 3:\n            x_smooth = signal.savgol_filter(x_raw, window_size, 2, mode='nearest')\n            y_smooth = signal.savgol_filter(y_raw, window_size, 2, mode='nearest')\n        else:\n            x_smooth = x_raw.copy()\n            y_smooth = y_raw.copy()\n        \n        # Calculate movement features\n        dx = np.diff(x_smooth, prepend=x_smooth[0]) * conversion_factor\n        dy = np.diff(y_smooth, prepend=y_smooth[0]) * conversion_factor\n        \n        # Speed calculation\n        distances = np.sqrt(dx**2 + dy**2)\n        speeds = distances * fps\n        \n        # Acceleration\n        accelerations = np.diff(speeds, prepend=speeds[0]) * fps\n        \n        # Direction and angular velocity\n        directions = np.arctan2(dy, dx) * 180 / np.pi\n        angular_velocities = np.diff(directions, prepend=directions[0])\n        # Handle angle wrapping\n        angular_velocities = np.where(angular_velocities > 180, angular_velocities - 360, angular_velocities)\n        angular_velocities = np.where(angular_velocities < -180, angular_velocities + 360, angular_velocities)\n        \n        # Distance to center (assuming center is at mean position)\n        center_x, center_y = np.nanmean(x_smooth), np.nanmean(y_smooth)\n        distances_to_center = np.sqrt((x_smooth - center_x)**2 + (y_smooth - center_y)**2) * conversion_factor\n        \n        # Update DataFrame\n        df.loc[indices, \"speed_cm_s\"] = speeds\n        df.loc[indices, \"accel_cm_s2\"] = accelerations\n        df.loc[indices, \"direction_deg\"] = directions\n        df.loc[indices, \"angular_velocity\"] = angular_velocities\n        df.loc[indices, \"distance_to_center\"] = distances_to_center\n        df.loc[indices, \"smoothed_x\"] = x_smooth\n        df.loc[indices, \"smoothed_y\"] = y_smooth\n    \n    return df\n\n# -------------------- Advanced Behavior Detection --------------------\ndef calculate_interaction_angle(agent_pos: Tuple[float, float], target_pos: Tuple[float, float], \n                              agent_direction: float) -> float:\n    \"\"\"Calculate the angle of interaction between agent and target\"\"\"\n    dx = target_pos[0] - agent_pos[0]\n    dy = target_pos[1] - agent_pos[1]\n    target_bearing = np.arctan2(dy, dx) * 180 / np.pi\n    \n    angle_diff = abs(agent_direction - target_bearing)\n    if angle_diff > 180:\n        angle_diff = 360 - angle_diff\n    \n    return angle_diff\n\ndef detect_complex_behaviors(centroids: pd.DataFrame, fps: float, pix_per_cm: Optional[float], \n                           params: EnhancedParams, video_id: int) -> List[Tuple]:\n    \"\"\"Enhanced behavior detection with all 30 behaviors and improved accuracy\"\"\"\n    results = []\n    \n    if centroids.empty:\n        return results\n    \n    # Prepare data\n    df = centroids.copy()\n    mouse_ids = sorted(df[\"mouse_id\"].dropna().unique())\n    \n    if len(mouse_ids) < 1:\n        return results\n    \n    # Create frame-aligned data\n    frame_range = range(int(df[\"video_frame\"].min()), int(df[\"video_frame\"].max()) + 1)\n    \n    # Distance conversion factor\n    distance_factor = pix_per_cm if pix_per_cm and pix_per_cm > 0 else 1.0\n    \n    # Behavior detection for each mouse pair\n    for agent_id in mouse_ids:\n        agent_data = df[df[\"mouse_id\"] == agent_id].set_index(\"video_frame\").sort_index()\n        \n        # Single mouse behaviors\n        single_behaviors = detect_single_mouse_behaviors(agent_data, fps, params, video_id, agent_id)\n        results.extend(single_behaviors)\n        \n        # Interactive behaviors\n        for target_id in mouse_ids:\n            if agent_id == target_id:\n                continue\n                \n            target_data = df[df[\"mouse_id\"] == target_id].set_index(\"video_frame\").sort_index()\n            interactive_behaviors = detect_interactive_behaviors(\n                agent_data, target_data, fps, distance_factor, params, video_id, agent_id, target_id\n            )\n            results.extend(interactive_behaviors)\n    \n    return results\n\ndef detect_single_mouse_behaviors(mouse_data: pd.DataFrame, fps: float, params: EnhancedParams, \n                                video_id: int, mouse_id: int) -> List[Tuple]:\n    \"\"\"Detect single mouse behaviors\"\"\"\n    behaviors = []\n    \n    if mouse_data.empty:\n        return behaviors\n    \n    speed = mouse_data[\"speed_cm_s\"].fillna(0)\n    accel = mouse_data[\"accel_cm_s2\"].fillna(0)\n    angular_vel = mouse_data[\"angular_velocity\"].fillna(0)\n    \n    frames = mouse_data.index.values\n    \n    # Behavior detection with improved thresholds\n    behavior_masks = {\n        \"rest\": speed < params.rest_speed_cm_s,\n        \"run\": speed > params.run_speed_cm_s,\n        \"freeze\": (speed < params.freeze_max_speed_cm_s) & (np.abs(accel) < 1.0),\n        \"rear\": (speed < params.rest_speed_cm_s) & (np.abs(angular_vel) > 30),\n        \"dig\": (speed < params.rest_speed_cm_s) & (np.abs(accel) > params.acceleration_threshold),\n        \"selfgroom\": (speed < params.rest_speed_cm_s) & (np.abs(angular_vel) > 45),\n        \"climb\": speed > params.run_speed_cm_s * 0.7,  # Moderate speed movement\n    }\n    \n    # Minimum durations in frames\n    min_durations = {\n        \"rest\": int(params.rest_min_duration * fps),\n        \"run\": int(params.run_min_duration * fps),\n        \"freeze\": int(params.freeze_min_duration * fps),\n        \"rear\": int(params.rear_min_duration * fps),\n        \"dig\": int(params.dig_min_duration * fps),\n        \"selfgroom\": int(params.selfgroom_min_duration * fps),\n        \"climb\": int(params.climb_min_duration * fps),\n    }\n    \n    # Extract behavior segments\n    for behavior, mask in behavior_masks.items():\n        min_dur = min_durations.get(behavior, int(params.min_duration_seconds * fps))\n        segments = extract_behavior_segments(mask.values, min_dur)\n        \n        for start_idx, end_idx in segments:\n            start_frame = frames[start_idx]\n            end_frame = frames[end_idx]\n            behaviors.append((video_id, mouse_id, mouse_id, behavior, start_frame, end_frame))\n    \n    return behaviors\n\ndef detect_interactive_behaviors(agent_data: pd.DataFrame, target_data: pd.DataFrame, \n                               fps: float, distance_factor: float, params: EnhancedParams,\n                               video_id: int, agent_id: int, target_id: int) -> List[Tuple]:\n    \"\"\"Detect interactive behaviors between two mice\"\"\"\n    behaviors = []\n    \n    # Align data by frame\n    common_frames = agent_data.index.intersection(target_data.index)\n    if len(common_frames) < 2:\n        return behaviors\n    \n    agent_aligned = agent_data.loc[common_frames]\n    target_aligned = target_data.loc[common_frames]\n    \n    # Calculate distances\n    dx = agent_aligned[\"smoothed_x\"] - target_aligned[\"smoothed_x\"]\n    dy = agent_aligned[\"smoothed_y\"] - target_aligned[\"smoothed_y\"]\n    distances = np.sqrt(dx**2 + dy**2) / distance_factor  # Convert to cm\n    \n    # Calculate relative speeds and directions\n    agent_speed = agent_aligned[\"speed_cm_s\"].fillna(0)\n    target_speed = target_aligned[\"speed_cm_s\"].fillna(0)\n    agent_direction = agent_aligned[\"direction_deg\"].fillna(0)\n    \n    # Interactive behavior detection\n    behavior_conditions = {\n        \"sniff\": (distances <= params.sniff_distance_cm) & (agent_speed < params.approach_max_speed_cm_s),\n        \"approach\": (distances <= params.approach_distance_cm) & (agent_speed > params.rest_speed_cm_s) & (agent_speed < params.run_speed_cm_s),\n        \"chase\": (distances <= params.chase_distance_cm) & (agent_speed > params.chase_speed_cm_s),\n        \"attack\": (distances <= params.attack_distance_cm) & (agent_speed > params.approach_max_speed_cm_s),\n        \"mount\": (distances <= params.mount_distance_cm) & (agent_speed < params.rest_speed_cm_s),\n        \"avoid\": (distances <= params.avoid_distance_cm) & (agent_speed > params.escape_speed_cm_s),\n        \"follow\": (distances <= params.follow_distance_cm) & (agent_speed > params.follow_speed_cm_s) & (agent_speed < params.run_speed_cm_s),\n        \"huddle\": (distances <= params.huddle_distance_cm) & (agent_speed < params.rest_speed_cm_s) & (target_speed < params.rest_speed_cm_s),\n        \"tussle\": (distances <= params.tussle_distance_cm) & (agent_speed > params.rest_speed_cm_s) & (target_speed > params.rest_speed_cm_s),\n    }\n    \n    # Additional complex behaviors\n    behavior_conditions.update({\n        \"allogroom\": (distances <= params.allogroom_distance_cm) & (agent_speed < params.rest_speed_cm_s),\n        \"shepherd\": (distances <= params.shepherd_distance_cm) & (agent_speed > params.shepherd_speed_cm_s),\n        \"disengage\": (distances <= params.disengage_distance_cm) & (agent_speed > params.escape_speed_cm_s),\n        \"defend\": (distances <= params.defend_distance_cm) & (agent_speed < params.approach_max_speed_cm_s),\n    })\n    \n    # Minimum durations\n    min_durations = {\n        \"sniff\": int(params.sniff_min_duration * fps),\n        \"approach\": int(params.approach_min_duration * fps),\n        \"chase\": int(params.chase_min_duration * fps),\n        \"attack\": int(params.attack_min_duration * fps),\n        \"mount\": int(params.mount_min_duration * fps),\n        \"avoid\": int(params.escape_min_duration * fps),\n        \"follow\": int(params.min_duration_seconds * fps),\n        \"huddle\": int(params.huddle_min_duration * fps),\n        \"tussle\": int(params.min_duration_seconds * fps),\n        \"allogroom\": int(params.allogroom_min_duration * fps),\n        \"shepherd\": int(params.min_duration_seconds * fps),\n        \"disengage\": int(params.min_duration_seconds * fps),\n        \"defend\": int(params.min_duration_seconds * fps),\n    }\n    \n    # Extract behavior segments\n    frames = common_frames.values\n    for behavior, condition in behavior_conditions.items():\n        min_dur = min_durations.get(behavior, int(params.min_duration_seconds * fps))\n        segments = extract_behavior_segments(condition.values, min_dur)\n        \n        for start_idx, end_idx in segments:\n            start_frame = frames[start_idx]\n            end_frame = frames[end_idx]\n            behaviors.append((video_id, agent_id, target_id, behavior, start_frame, end_frame))\n    \n    return behaviors\n\ndef extract_behavior_segments(mask: np.ndarray, min_length: int) -> List[Tuple[int, int]]:\n    \"\"\"Extract continuous segments from boolean mask with minimum length\"\"\"\n    if len(mask) == 0:\n        return []\n    \n    # Convert to boolean array\n    mask = np.asarray(mask, dtype=bool)\n    \n    if not np.any(mask):\n        return []\n    \n    # Find transitions\n    padded = np.concatenate(([False], mask, [False]))\n    diff = np.diff(padded.astype(int))\n    \n    starts = np.where(diff == 1)[0]\n    ends = np.where(diff == -1)[0] - 1\n    \n    # Filter by minimum length\n    segments = []\n    for start, end in zip(starts, ends):\n        if (end - start + 1) >= min_length:\n            segments.append((start, end))\n    \n    return segments\n\n# -------------------- Dataset Discovery and Processing --------------------\ndef find_dataset_root() -> str:\n    \"\"\"Enhanced dataset discovery\"\"\"\n    base_paths = [\"/kaggle/input\", \"./data\", \"../input\", \".\"]\n    \n    for base in base_paths:\n        if not os.path.exists(base):\n            continue\n            \n        candidates = []\n        \n        for item in os.listdir(base):\n            path = os.path.join(base, item)\n            if not os.path.isdir(path):\n                continue\n                \n            # Check for required files\n            has_train = os.path.exists(os.path.join(path, \"train.csv\"))\n            has_test = os.path.exists(os.path.join(path, \"test.csv\"))\n            has_train_tracking = os.path.exists(os.path.join(path, \"train_tracking\"))\n            has_test_tracking = os.path.exists(os.path.join(path, \"test_tracking\"))\n            \n            if (has_train or has_test) and (has_train_tracking or has_test_tracking):\n                candidates.append(path)\n                continue\n                \n            # Check subdirectories\n            try:\n                for sub in os.listdir(path):\n                    subpath = os.path.join(path, sub)\n                    if not os.path.isdir(subpath):\n                        continue\n                        \n                    has_train = os.path.exists(os.path.join(subpath, \"train.csv\"))\n                    has_test = os.path.exists(os.path.join(subpath, \"test.csv\"))\n                    has_train_tracking = os.path.exists(os.path.join(subpath, \"train_tracking\"))\n                    has_test_tracking = os.path.exists(os.path.join(subpath, \"test_tracking\"))\n                    \n                    if (has_train or has_test) and (has_train_tracking or has_test_tracking):\n                        candidates.append(subpath)\n                        break\n            except PermissionError:\n                continue\n        \n        if candidates:\n            return candidates[0]\n    \n    return os.getcwd()\n\ndef read_metadata_enhanced(csv_path: str) -> pd.DataFrame:\n    \"\"\"Enhanced metadata reading with better error handling\"\"\"\n    try:\n        df = pd.read_csv(csv_path)\n    except Exception as e:\n        print(f\"Error reading metadata from {csv_path}: {e}\")\n        return pd.DataFrame()\n    \n    # Standardize column names\n    column_mappings = {\n        \"fps\": [\"frames per second\", \"frames_per_second\", \"frame_rate\", \"framerate\"],\n        \"pix_per_cm\": [\"pix per cm (approx)\", \"pix_per_cm_approx\", \"pixels_per_cm\", \"pixel_per_cm\"]\n    }\n    \n    for standard_name, alternatives in column_mappings.items():\n        if standard_name not in df.columns:\n            for alt in alternatives:\n                if alt in df.columns:\n                    df = df.rename(columns={alt: standard_name})\n                    break\n    \n    # Set default values for missing columns\n    if \"fps\" not in df.columns:\n        df[\"fps\"] = 30.0  # Default FPS\n        print(\"Warning: No FPS information found, using default value of 30 FPS\")\n    \n    if \"pix_per_cm\" not in df.columns:\n        df[\"pix_per_cm\"] = 10.0  # Default pixels per cm\n        print(\"Warning: No pixels per cm information found, using default value of 10\")\n    \n    # Clean and validate data\n    df[\"fps\"] = pd.to_numeric(df[\"fps\"], errors='coerce').fillna(30.0)\n    df[\"pix_per_cm\"] = pd.to_numeric(df[\"pix_per_cm\"], errors='coerce').fillna(10.0)\n    \n    return df\n\ndef load_tracking_data_enhanced(root_dir: str, split: str, lab_id: str, video_id: int) -> pd.DataFrame:\n    \"\"\"Enhanced tracking data loading with better error handling\"\"\"\n    parquet_path = os.path.join(root_dir, f\"{split}_tracking\", str(lab_id), f\"{int(video_id)}.parquet\")\n    \n    if not os.path.exists(parquet_path):\n        # Try alternative file extensions and naming conventions\n        alternatives = [\n            os.path.join(root_dir, f\"{split}_tracking\", str(lab_id), f\"{int(video_id)}.csv\"),\n            os.path.join(root_dir, f\"{split}_tracking\", f\"{lab_id}_{int(video_id)}.parquet\"),\n            os.path.join(root_dir, f\"{split}_tracking\", f\"{int(video_id)}.parquet\"),\n        ]\n        \n        for alt_path in alternatives:\n            if os.path.exists(alt_path):\n                parquet_path = alt_path\n                break\n        else:\n            raise FileNotFoundError(f\"No tracking data found for video {video_id}\")\n    \n    try:\n        if parquet_path.endswith('.parquet'):\n            df = pd.read_parquet(parquet_path)\n        else:\n            df = pd.read_csv(parquet_path)\n        \n        return _standardize_tracking_columns(df)\n    except Exception as e:\n        print(f\"Error loading tracking data from {parquet_path}: {e}\")\n        return pd.DataFrame()\n\n# -------------------- Main Processing Pipeline --------------------\ndef build_enhanced_submission(root_dir: str, split: str, output_path: str, params: EnhancedParams):\n    \"\"\"Enhanced submission building with improved error handling and validation\"\"\"\n    meta_csv = os.path.join(root_dir, f\"{split}.csv\")\n    \n    if not os.path.exists(meta_csv):\n        raise FileNotFoundError(f\"Metadata file not found: {meta_csv}\")\n    \n    meta_df = read_metadata_enhanced(meta_csv)\n    if meta_df.empty:\n        raise ValueError(\"No metadata loaded\")\n    \n    print(f\"Processing {len(meta_df)} videos for {split} split...\")\n    \n    all_events = []\n    processing_errors = 0\n    \n    for idx, row in tqdm(meta_df.iterrows(), total=len(meta_df), desc=f\"Processing {split}\"):\n        try:\n            video_id = int(row[\"video_id\"])\n            lab_id = str(row[\"lab_id\"])\n            fps = float(row[\"fps\"]) if pd.notna(row[\"fps\"]) else 30.0\n            pix_per_cm = float(row[\"pix_per_cm\"]) if pd.notna(row[\"pix_per_cm\"]) else 10.0\n            \n            # Load tracking data\n            tracking_data = load_tracking_data_enhanced(root_dir, split, lab_id, video_id)\n            \n            if tracking_data.empty:\n                print(f\"Warning: Empty tracking data for video {video_id}\")\n                continue\n            \n            # Compute features\n            centroids = compute_winner_features(tracking_data)\n            if centroids.empty:\n                continue\n                \n            enhanced_centroids = enhanced_movement_features(centroids, fps, pix_per_cm, params)\n            \n            # Detect behaviors\n            video_events = detect_complex_behaviors(enhanced_centroids, fps, pix_per_cm, params, video_id)\n            all_events.extend(video_events)\n            \n        except Exception as e:\n            processing_errors += 1\n            print(f\"Error processing video {video_id}: {e}\")\n            continue\n    \n    print(f\"Processing complete. Errors: {processing_errors}/{len(meta_df)}\")\n    print(f\"Total events detected: {len(all_events)}\")\n    \n    # Create submission dataframe\n    if not all_events:\n        print(\"Warning: No events detected! Creating empty submission.\")\n        submission_df = pd.DataFrame(columns=[\"row_id\", \"video_id\", \"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"])\n    else:\n        rows = []\n        for i, (vid, agent, target, action, start, end) in enumerate(all_events):\n            rows.append({\n                \"row_id\": i,\n                \"video_id\": vid,\n                \"agent_id\": str(agent),\n                \"target_id\": str(target),\n                \"action\": action,\n                \"start_frame\": int(start),\n                \"stop_frame\": int(end)\n            })\n        \n        submission_df = pd.DataFrame(rows)\n    \n    # Validate submission format\n    required_columns = [\"row_id\", \"video_id\", \"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"]\n    for col in required_columns:\n        if col not in submission_df.columns:\n            print(f\"Error: Missing required column {col}\")\n            return\n    \n    # Save submission\n    submission_df.to_csv(output_path, index=False)\n    print(f\"✅ Enhanced submission saved to {output_path} with {len(submission_df)} events\")\n    \n    # Print behavior summary\n    if not submission_df.empty:\n        behavior_counts = submission_df[\"action\"].value_counts()\n        print(\"\\nBehavior distribution:\")\n        for behavior, count in behavior_counts.head(10).items():\n            print(f\"  {behavior}: {count}\")\n\ndef compute_winner_features(tracking: pd.DataFrame) -> pd.DataFrame:\n    \"\"\"Compute centroid features from tracking data\"\"\"\n    if tracking.empty:\n        return pd.DataFrame()\n    \n    # Group by frame and mouse to get centroids\n    centroids = tracking.groupby([\"video_frame\", \"mouse_id\"], as_index=False)[[\"x\", \"y\"]].mean()\n    centroids = centroids.rename(columns={\"x\": \"centroid_x\", \"y\": \"centroid_y\"})\n    \n    return centroids\n\n# -------------------- Main Execution --------------------\nif __name__ == \"__main__\":\n    # Find dataset\n    ROOT = find_dataset_root()\n    print(f\"Using dataset root: {ROOT}\")\n    \n    # Enhanced parameters\n    params = EnhancedParams()\n    \n    # Build enhanced submission\n    output_csv = \"submission.csv\"\n    \n    try:\n        build_enhanced_submission(ROOT, \"test\", output_csv, params)\n        print(\"✅ Enhanced submission completed successfully!\")\n    except Exception as e:\n        print(f\"❌ Error building submission: {e}\")\n        \n    # Optional: Create a sample visualization\n    try:\n        meta = read_metadata_enhanced(os.path.join(ROOT, \"train.csv\"))\n        if not meta.empty:\n            print(\"Creating sample visualization...\")\n            row0 = meta.iloc[0]\n            # Add visualization code here if needed\n    except Exception as e:\n        print(f\"Could not create visualization: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-27T17:28:43.154640Z","iopub.execute_input":"2025-09-27T17:28:43.154908Z","iopub.status.idle":"2025-09-27T17:28:43.649165Z","shell.execute_reply.started":"2025-09-27T17:28:43.154890Z","shell.execute_reply":"2025-09-27T17:28:43.648430Z"}},"outputs":[],"execution_count":null}]}