{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n         print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# setup","metadata":{}},{"cell_type":"code","source":"!pip install numpy pandas matplotlib","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:08.850195Z","iopub.execute_input":"2025-12-11T05:57:08.850534Z","iopub.status.idle":"2025-12-11T05:57:13.055555Z","shell.execute_reply.started":"2025-12-11T05:57:08.850516Z","shell.execute_reply":"2025-12-11T05:57:13.054676Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install lightgbm","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:13.056582Z","iopub.execute_input":"2025-12-11T05:57:13.056825Z","iopub.status.idle":"2025-12-11T05:57:16.072011Z","shell.execute_reply.started":"2025-12-11T05:57:13.056796Z","shell.execute_reply":"2025-12-11T05:57:16.071312Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Explore datasets","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\ndef inspect_parquet(path, frame_to_plot=0, figsize=(8,6)):\n    df = pd.read_parquet(path)\n    print(\"Loaded:\", path)\n    print(\"Shape:\", df.shape)\n    print(\"Columns:\", list(df.columns))\n    print(\"\\nHead:\")\n    display(df.head())\n\n    # Heuristic: tracking/pose file contains 'x' and 'y' and 'bodypart' columns\n    if {'x','y','bodypart','video_frame','mouse_id'}.issubset(df.columns):\n        print(\"\\nDetected: POSE / TRACKING file\")\n        print(\"Unique mice:\", sorted(df['mouse_id'].unique()))\n        print(\"Keypoints:\", sorted(df['bodypart'].unique()))\n        print(\"Number of frames:\", int(df['video_frame'].nunique()))\n        frame = frame_to_plot\n        print(f\"\\nPlotting pose for frame {frame} (frame index may start at 0)\")\n\n        plt.figure(figsize=figsize)\n        for mid in sorted(df['mouse_id'].unique()):\n            sub = df[(df['mouse_id']==mid) & (df['video_frame']==frame)]\n            if sub.empty:\n                print(f\"  Warning: mouse {mid} has no keypoints at frame {frame}\")\n                continue\n            # scatter keypoints\n            plt.scatter(sub['x'], sub['y'], label=f\"mouse {mid}\", s=20)\n            # annotate keypoints with their name for clarity\n            for _, row in sub.iterrows():\n                bp = str(row['bodypart'])\n                plt.text(row['x']+2, row['y']+2, bp, fontsize=6)\n        plt.gca().invert_yaxis()\n        plt.legend()\n        plt.title(f\"Pose at frame {frame}\")\n        plt.xlabel(\"x (pixels)\")\n        plt.ylabel(\"y (pixels)\")\n        plt.tight_layout()\n        plt.show()\n\n        max_frame = int(df['video_frame'].max())\n        print(\"Frame range: 0 ..\", max_frame)\n\n    elif {'agent_id','target_id','action','start_frame','stop_frame'}.issubset(df.columns):\n        print(\"\\nDetected: ANNOTATION file\")\n        print(\"Unique agents:\", sorted(df['agent_id'].unique()))\n        print(\"Unique targets:\", sorted(df['target_id'].unique()))\n        print(\"Unique actions:\", sorted(df['action'].unique()))\n        print(\"Number of annotated segments:\", len(df))\n        # Convert start/stop to ints (safety)\n        df['start_frame'] = df['start_frame'].astype(int)\n        df['stop_frame'] = df['stop_frame'].astype(int)\n\n        # Basic statistics\n        df['duration'] = df['stop_frame'] - df['start_frame'] + 1\n        print(\"\\nPer-action counts and mean durations:\")\n        print(df.groupby('action')['duration'].agg(['count','mean']).sort_values('count', ascending=False))\n\n        # Gantt-style timeline plot\n        agents = sorted(df['agent_id'].unique())\n        actions = sorted(df['action'].unique())\n        action_to_color = {a: i for i, a in enumerate(actions)} \n\n        fig, ax = plt.subplots(figsize=(12, max(2, 0.5*len(agents))))\n        yticks = []\n        yticklabels = []\n        for i, agent in enumerate(agents):\n            agent_segs = df[df['agent_id']==agent]\n            y = i\n            yticks.append(y)\n            yticklabels.append(f\"agent {agent}\")\n            for _, row in agent_segs.iterrows():\n                start = int(row['start_frame'])\n                stop = int(row['stop_frame'])\n                action = row['action']\n                color_idx = action_to_color[action]\n                ax.barh(y, width=(stop - start + 1), left=start, height=0.6,\n                        color=plt.cm.tab20(color_idx % 20), edgecolor='k', alpha=0.8)\n                if (stop - start) > 10:\n                    ax.text(start + 1, y, str(action), va='center', ha='left', fontsize=7, color='white')\n\n        ax.set_yticks(yticks)\n        ax.set_yticklabels(yticklabels)\n        ax.set_xlabel(\"frame\")\n        ax.set_title(\"Annotation timeline (per agent)\")\n        ax.invert_yaxis()\n        # legend\n        handles = []\n        labels = []\n        for a, idx in action_to_color.items():\n            handles.append(plt.Rectangle((0,0),1,1, color=plt.cm.tab20(idx % 20)))\n            labels.append(a)\n        ax.legend(handles, labels, bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=8)\n        plt.tight_layout()\n        plt.show()\n\n        print(\"\\nPer-agent annotated time (frames):\")\n        print(df.groupby('agent_id')['duration'].sum())\n\n    else:\n        print(\"\\nUnknown parquet format. Columns are not recognized as pose or annotation.\")\n        print(\"Example columns:\", list(df.columns))\n        print(\"\\nIf this is a tracking file, it should contain at least: x, y, bodypart, video_frame, mouse_id\")\n        print(\"If this is an annotation file, it should contain at least: agent_id, target_id, action, start_frame, stop_frame\")\n    return df\n\npath = \"/kaggle/input/MABe-mouse-behavior-detection/train_annotation/AdaptableSnail/1212811043.parquet\"\n\n\ndf = inspect_parquet(path, frame_to_plot=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:16.073096Z","iopub.execute_input":"2025-12-11T05:57:16.073378Z","iopub.status.idle":"2025-12-11T05:57:17.434725Z","shell.execute_reply.started":"2025-12-11T05:57:16.073354Z","shell.execute_reply":"2025-12-11T05:57:17.434088Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"| Column            | Meaning                                                         | Example                               | Explanation                                                                                   |\n| ----------------- | --------------------------------------------------------------- | ------------------------------------- | --------------------------------------------------------------------------------------------- |\n| **`video_frame`** | Frame index in the video                                        | `0`, `1`, `2`, …                      | Each frame is a single time step in the video sequence.                                       |\n| **`mouse_id`**    | Which mouse this data belongs to                                | `1`, `2`, `3`, …                      | If there are multiple mice in the arena, each gets a unique ID (like “mouse 1” or “mouse 2”). |\n| **`bodypart`**    | The specific tracked keypoint on that mouse’s body              | `nose`, `ear_left`, `tail_base`, etc. | These are the “landmarks” used for motion tracking.                                           |\n| **`x`**           | Horizontal coordinate of that bodypart in the frame (in pixels) | `909.57`                              | How far right this point is in the frame.                                                     |\n| **`y`**           | Vertical coordinate of that bodypart (in pixels)                | `960.27`                              | How far down this point is in the frame.                                                      |\n","metadata":{}},{"cell_type":"markdown","source":"## sturcture of the dataset file","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom pathlib import Path\n\n# Base dataset path\nBASE_PATH = Path(\"/kaggle/input/MABe-mouse-behavior-detection\")\n\n# Function to peek into a parquet file\ndef peek_parquet(file_path, n_rows=5):\n    try:\n        df = pd.read_parquet(file_path)\n        print(f\"\\nFile: {file_path.name}\")\n        print(\"Shape:\", df.shape)\n        print(\"Columns:\", list(df.columns))\n        display(df.head(n_rows))\n\n        # Check if pose/tracking or annotation file\n        if {'x','y','bodypart','video_frame','mouse_id'}.issubset(df.columns):\n            print(\"Type: POSE / TRACKING file\")\n            print(\"Unique mice:\", df['mouse_id'].unique())\n            print(\"Unique bodyparts:\", df['bodypart'].unique())\n            print(\"Number of frames:\", df['video_frame'].nunique())\n        elif {'agent_id','target_id','action','start_frame','stop_frame'}.issubset(df.columns):\n            print(\"Type: ANNOTATION file\")\n            print(\"Unique agents:\", df['agent_id'].unique())\n            print(\"Unique targets:\", df['target_id'].unique())\n            print(\"Unique actions:\", df['action'].unique())\n            print(\"Number of annotated segments:\", len(df))\n        else:\n            print(\"Unknown file type\")\n        return df\n    except Exception as e:\n        print(f\"Could not read {file_path.name}: {e}\")\n        return None\n\n# 1. List all directories and files\nprint(\"Dataset structure:\")\nfor root, dirs, files in os.walk(BASE_PATH):\n    level = root.replace(str(BASE_PATH), '').count(os.sep)\n    indent = ' ' * 4 * level\n    print(f\"{indent}{os.path.basename(root)}/\")\n    subindent = ' ' * 4 * (level + 1)\n    for f in files[:5]:  # print first 5 files in each folder\n        print(f\"{subindent}{f}\")\n    if len(files) > 5:\n        print(f\"{subindent}... ({len(files)-5} more files)\")\n\n# 2. Count files per type\nfile_counts = {}\nfor path in BASE_PATH.rglob(\"*.parquet\"):\n    parent = path.parent.name\n    file_counts[parent] = file_counts.get(parent, 0) + 1\n\nprint(\"\\nNumber of parquet files per folder:\")\nfor k,v in file_counts.items():\n    print(f\"{k}: {v}\")\n\n# 3. Peek into a few files\nprint(\"\\nPeeking into some train annotation files:\")\ntrain_ann_path = BASE_PATH / \"train_annotation\" / \"AdaptableSnail\"\ntrain_ann_files = list(train_ann_path.glob(\"*.parquet\"))\nfor f in train_ann_files[:2]:  # peek first 2 files\n    peek_parquet(f)\n\nprint(\"\\nPeeking into some test tracking files:\")\ntest_track_path = BASE_PATH / \"test_tracking\" / \"AdaptableSnail\"\ntest_track_files = list(test_track_path.glob(\"*.parquet\"))\nfor f in test_track_files[:2]:  # peek first 2 files\n    peek_parquet(f)\n\n# 4. Optional: summary stats across dataset\nprint(\"\\nSummary across all train annotation files:\")\nall_actions = []\nfor f in train_ann_files:\n    df = peek_parquet(f)\n    if df is not None and 'action' in df.columns:\n        all_actions.extend(df['action'].unique())\nprint(\"All unique actions in train:\", set(all_actions))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:17.435491Z","iopub.execute_input":"2025-12-11T05:57:17.435703Z","iopub.status.idle":"2025-12-11T05:57:20.940995Z","shell.execute_reply.started":"2025-12-11T05:57:17.435687Z","shell.execute_reply":"2025-12-11T05:57:20.940403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/MABe-mouse-behavior-detection/test.csv\")\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:20.941658Z","iopub.execute_input":"2025-12-11T05:57:20.941901Z","iopub.status.idle":"2025-12-11T05:57:20.966851Z","shell.execute_reply.started":"2025-12-11T05:57:20.941885Z","shell.execute_reply":"2025-12-11T05:57:20.966184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/MABe-mouse-behavior-detection/train.csv\")\ndf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T05:57:20.968586Z","iopub.execute_input":"2025-12-11T05:57:20.968972Z","iopub.status.idle":"2025-12-11T05:57:21.067013Z","shell.execute_reply.started":"2025-12-11T05:57:20.968950Z","shell.execute_reply":"2025-12-11T05:57:21.066386Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Mouse Behavior Detection Model\n","metadata":{}},{"cell_type":"code","source":"import os\nimport glob\nimport math\nimport gc\nimport joblib\nimport warnings\nfrom collections import deque\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report, accuracy_score\nimport lightgbm as lgb\n\nwarnings.filterwarnings(\"ignore\")\n\n# ---------------------------\n# CONFIG\n# ---------------------------\nCONFIG = {\n    'input_path': '/kaggle/input/MABe-mouse-behavior-detection',  # set to your input dir\n    'processed_data_path': 'train_features.parquet',\n    'model_save_path': 'lgbm_model.pkl',\n    'encoder_save_path': 'label_encoder.pkl',\n    'train_limit_per_lab': 30,           # how many videos per lab to use (limits runtime)\n    'frame_col': 'video_frame',\n    'chunk_prefix': 'chunk_',            # temporary chunk files\n    'max_chunks_to_load': 8,             # safety for memory when loading chunk files\n    'min_segment_frames': 3,             # min frames to keep a predicted segment\n    'smoothing_window': 5,               # majority smoothing window (odd integer)\n    'submission_name': 'submission.csv'\n}\n\n# ---------------------------\n# FEATURE EXTRACTOR (robust to bodypart names)\n# ---------------------------\ndef robust_pivot_tracking(df_tracking):\n    \"\"\"\n    Convert tracking dataframe to wide format with columns like:\n    <mouseid>_<bodypart>_x, <mouseid>_<bodypart>_y\n    Returns df_wide with frame column named CONFIG['frame_col']\n    \"\"\"\n    if 'frame' in df_tracking.columns and CONFIG['frame_col'] not in df_tracking.columns:\n        df_tracking = df_tracking.rename(columns={'frame': CONFIG['frame_col']})\n    # Expect columns at least: [video_frame, mouse_id, bodypart, x, y]\n    if not {'mouse_id','bodypart','x','y'}.issubset(set(df_tracking.columns)):\n        # Try alternative names\n        possible = set(df_tracking.columns)\n        # fail gracefully\n        return None\n\n    # pivot\n    try:\n        df_wide = df_tracking.pivot_table(\n            index=CONFIG['frame_col'],\n            columns=['mouse_id','bodypart'],\n            values=['x','y'],\n            aggfunc='first'\n        )\n    except Exception:\n        return None\n\n    # flatten\n    df_wide.columns = [f\"{mid}_{bp}_{coord}\" for coord, mid, bp in df_wide.columns]\n    df_wide = df_wide.reset_index()\n    return df_wide\n\ndef find_part_cols(df_wide, mouse, part_names):\n    \"\"\"Return (x_col, y_col) for the first matching part in part_names for given mouse\"\"\"\n    for part in part_names:\n        x_col = f\"{mouse}_{part}_x\"\n        y_col = f\"{mouse}_{part}_y\"\n        if x_col in df_wide.columns and y_col in df_wide.columns:\n            return x_col, y_col\n    # try fuzzy search\n    for c in df_wide.columns:\n        if c.startswith(f\"{mouse}_\") and any(p in c for p in part_names) and c.endswith('_x'):\n            return c, c[:-2] + '_y'\n    return None, None\n\ndef angle_between_points(x1, y1, x2, y2):\n    # angle of vector tail->head\n    return np.arctan2(y1 - y2, x1 - x2)\n\ndef extract_features_from_tracking(df_tracking):\n    \"\"\"\n    Input: raw tracking dataframe (per-video, possibly many frames)\n    Output: dataframe with frame column and engineered features\n    \"\"\"\n    try:\n        df_wide = robust_pivot_tracking(df_tracking)\n        if df_wide is None:\n            return None\n    except Exception:\n        return None\n\n    # Ensure frame col present\n    if CONFIG['frame_col'] not in df_wide.columns:\n        return None\n\n    # find mouse ids present\n    mice_ids = sorted(set([col.split('_')[0] for col in df_wide.columns if col!=CONFIG['frame_col']]))\n    if len(mice_ids) < 2:\n        # pad zeros if only one mouse or missing\n        # return a df with required frame col and zeros\n        out = pd.DataFrame({CONFIG['frame_col']: df_wide[CONFIG['frame_col']]})\n        # add placeholder features so pipeline won't crash\n        out['dist_m1_m2'] = 0.0\n        out['m1_dx'] = out['m1_dy'] = out['m2_dx'] = out['m2_dy'] = 0.0\n        out['m1_speed'] = out['m2_speed'] = 0.0\n        out['rel_dx'] = out['rel_dy'] = 0.0\n        out['m1_angle'] = out['m2_angle'] = 0.0\n        out['angle_diff'] = 0.0\n        out['m1_acc'] = out['m2_acc'] = out['rel_acc'] = 0.0\n        out['ddist'] = 0.0\n        out['dist_nose_tail'] = 0.0\n        out['dist_head_head'] = 0.0\n        return out\n\n    # choose two mice: first two sorted ids\n    m1, m2 = mice_ids[0], mice_ids[1]\n\n    # find bodyparts robustly\n    m1_head_x, m1_head_y = find_part_cols(df_wide, m1, ['head','nose','snout'])\n    m1_tail_x, m1_tail_y = find_part_cols(df_wide, m1, ['tail','tail_base','root','spine','body','center'])\n    m2_head_x, m2_head_y = find_part_cols(df_wide, m2, ['head','nose','snout'])\n    m2_tail_x, m2_tail_y = find_part_cols(df_wide, m2, ['tail','tail_base','root','spine','body','center'])\n\n    # create output df starting from frame\n    out = pd.DataFrame({CONFIG['frame_col']: df_wide[CONFIG['frame_col']]})\n\n    # helper safe column fetch\n    def col_or_zero(col):\n        return df_wide[col] if (col and col in df_wide.columns) else pd.Series(0.0, index=df_wide.index)\n\n    # compute distances (use head/head fallback)\n    h1x = col_or_zero(m1_head_x)\n    h1y = col_or_zero(m1_head_y)\n    h2x = col_or_zero(m2_head_x)\n    h2y = col_or_zero(m2_head_y)\n\n    out['dist_m1_m2'] = np.sqrt((h1x - h2x)**2 + (h1y - h2y)**2).astype('float32')\n    out['dist_head_head'] = out['dist_m1_m2'].astype('float32')\n\n    # nose (for nose->tail)\n    m1_nose_x, m1_nose_y = find_part_cols(df_wide, m1, ['nose','snout','head'])\n    m2_nose_x, m2_nose_y = find_part_cols(df_wide, m2, ['nose','snout','head'])\n    m1_nose_x_s = col_or_zero(m1_nose_x)\n    m1_nose_y_s = col_or_zero(m1_nose_y)\n    m2_tail_x_s = col_or_zero(m2_tail_x)\n    m2_tail_y_s = col_or_zero(m2_tail_y)\n    out['dist_nose_tail'] = np.sqrt((m1_nose_x_s - m2_tail_x_s)**2 + (m1_nose_y_s - m2_tail_y_s)**2).astype('float32')\n\n    # directional velocities (head position diffs)\n    out['m1_dx'] = h1x.diff().fillna(0).astype('float32')\n    out['m1_dy'] = h1y.diff().fillna(0).astype('float32')\n    out['m2_dx'] = h2x.diff().fillna(0).astype('float32')\n    out['m2_dy'] = h2y.diff().fillna(0).astype('float32')\n\n    out['m1_speed'] = np.sqrt(out['m1_dx']**2 + out['m1_dy']**2).astype('float32')\n    out['m2_speed'] = np.sqrt(out['m2_dx']**2 + out['m2_dy']**2).astype('float32')\n\n    # relative motion\n    out['rel_dx'] = (out['m1_dx'] - out['m2_dx']).astype('float32')\n    out['rel_dy'] = (out['m1_dy'] - out['m2_dy']).astype('float32')\n    out['rel_speed'] = np.sqrt(out['rel_dx']**2 + out['rel_dy']**2).astype('float32')\n\n    # orientation angles (tail->head)\n    t1x = col_or_zero(m1_tail_x)\n    t1y = col_or_zero(m1_tail_y)\n    t2x = col_or_zero(m2_tail_x)\n    t2y = col_or_zero(m2_tail_y)\n\n    out['m1_angle'] = angle_between_points(h1x, h1y, t1x, t1y).astype('float32')\n    out['m2_angle'] = angle_between_points(h2x, h2y, t2x, t2y).astype('float32')\n\n    # normalized angle difference to [-pi, pi]\n    angle_diff = out['m1_angle'] - out['m2_angle']\n    out['angle_diff'] = ((angle_diff + np.pi) % (2 * np.pi) - np.pi).astype('float32')\n\n    # acceleration\n    out['m1_acc'] = out['m1_speed'].diff().fillna(0).astype('float32')\n    out['m2_acc'] = out['m2_speed'].diff().fillna(0).astype('float32')\n    out['rel_acc'] = out['rel_speed'].diff().fillna(0).astype('float32')\n\n    # distance change rate\n    out['ddist'] = out['dist_m1_m2'].diff().fillna(0).astype('float32')\n\n    # replace inf/nan\n    out = out.replace([np.inf, -np.inf], 0.0).fillna(0.0)\n\n    # keep an ordered feature set\n    feature_cols = [\n        'dist_m1_m2', 'dist_head_head', 'dist_nose_tail',\n        'm1_dx','m1_dy','m2_dx','m2_dy',\n        'm1_speed','m2_speed','rel_dx','rel_dy','rel_speed',\n        'm1_angle','m2_angle','angle_diff',\n        'm1_acc','m2_acc','rel_acc','ddist'\n    ]\n    # ensure all exist\n    for c in feature_cols:\n        if c not in out.columns:\n            out[c] = 0.0\n    return out[ [CONFIG['frame_col']] + feature_cols ]\n\n# ---------------------------\n# CHUNKED PROCESS & SAVE\n# ---------------------------\ndef process_and_save_data():\n    print(\">>> Phase 1: Processing Data in Chunks...\")\n    search_path = f\"{CONFIG['input_path']}/train_tracking/*/*.parquet\"\n    all_files = glob.glob(search_path)\n    lab_files = {}\n    for f in all_files:\n        lab_name = os.path.basename(os.path.dirname(f))\n        # optional: skip some labs if needed\n        if lab_name not in lab_files:\n            lab_files[lab_name] = []\n        lab_files[lab_name].append(f)\n\n    first_chunk = True\n    total_processed = 0\n    chunk_idx = 0\n\n    for lab, files in lab_files.items():\n        selected_files = files[:CONFIG['train_limit_per_lab']]\n        print(f\"   Lab {lab}: Processing {len(selected_files)} videos...\")\n        for trk_path in selected_files:\n            ann_path = trk_path.replace('train_tracking', 'train_annotation')\n            if not os.path.exists(ann_path):\n                continue\n            try:\n                df_trk = pd.read_parquet(trk_path)\n                df_ann = pd.read_parquet(ann_path)\n                X_df = extract_features_from_tracking(df_trk)\n                if X_df is None or X_df.empty:\n                    continue\n\n                # default labels = 'other'\n                y = pd.Series(['other'] * len(X_df), index=X_df.index, name='action')\n                # assign annotated actions (frame ranges)\n                for _, row in df_ann.iterrows():\n                    try:\n                        start = int(row['start_frame'])\n                        stop = int(row['stop_frame'])\n                        mask = (X_df[CONFIG['frame_col']] >= start) & (X_df[CONFIG['frame_col']] <= stop)\n                        y.loc[mask] = row['action']\n                    except Exception:\n                        continue\n\n                chunk_df = pd.concat([X_df, y], axis=1)\n                # drop frame col before saving to parquet for training accumulation\n                to_save = chunk_df.drop(columns=[CONFIG['frame_col']])\n                # downcast floats\n                float_cols = to_save.select_dtypes(include=['float64']).columns\n                to_save[float_cols] = to_save[float_cols].astype('float32')\n\n                if first_chunk:\n                    to_save.to_parquet(CONFIG['processed_data_path'], engine='pyarrow', index=False)\n                    first_chunk = False\n                else:\n                    fname = f\"{CONFIG['chunk_prefix']}{chunk_idx}.parquet\"\n                    to_save.to_parquet(fname, index=False)\n                    chunk_idx += 1\n\n                total_processed += 1\n                del df_trk, df_ann, X_df, chunk_df, to_save, y\n                gc.collect()\n            except Exception as e:\n                # skip problematic video\n                # print(\"skip\", trk_path, e)\n                continue\n\n    print(f\"Processed {total_processed} videos.\")\n    return total_processed\n\n# ---------------------------\n# TRAINING\n# ---------------------------\ndef train_model():\n    print(\">>> Phase 2: Training & Evaluation...\")\n    if not os.path.exists(CONFIG['processed_data_path']):\n        raise ValueError(\"No processed data found. Run process_and_save_data() first.\")\n\n    df_main = pd.read_parquet(CONFIG['processed_data_path'])\n    # load extra chunks\n    chunk_files = sorted(glob.glob(f\"{CONFIG['chunk_prefix']}*.parquet\"))[:CONFIG['max_chunks_to_load']]\n    for c in chunk_files:\n        temp = pd.read_parquet(c)\n        df_main = pd.concat([df_main, temp], axis=0)\n        del temp\n        gc.collect()\n\n    print(f\"Total Dataset Shape: {df_main.shape}\")\n\n    # feature / label split\n    feature_cols = [c for c in df_main.columns if c != 'action']\n    X = df_main[feature_cols]\n    y = df_main['action']\n\n    le = LabelEncoder()\n    y_enc = le.fit_transform(y)\n\n    # Stratified split\n    X_train, X_val, y_train, y_val = train_test_split(X, y_enc, test_size=0.2,\n                                                      random_state=42, stratify=y_enc)\n\n    print(\"  Fitting LightGBM with class balancing...\")\n    clf = lgb.LGBMClassifier(\n        n_estimators=300,\n        learning_rate=0.05,\n        n_jobs=-1,\n        class_weight='balanced',\n        objective='multiclass'\n    )\n    clf.fit(\n        X_train,\n        y_train,\n        eval_set=[(X_val, y_val)],\n        callbacks=[\n            lgb.early_stopping(stopping_rounds=30),\n            lgb.log_evaluation(50)\n        ]\n    )\n\n    # Evaluate\n    val_preds = clf.predict(X_val)\n    print(f\"\\nValidation Accuracy: {accuracy_score(y_val, val_preds):.4f}\")\n    try:\n        print(classification_report(y_val, val_preds, target_names=le.classes_))\n    except Exception:\n        print(classification_report(y_val, val_preds))\n\n    # Save model & encoder\n    joblib.dump(clf, CONFIG['model_save_path'])\n    joblib.dump(le, CONFIG['encoder_save_path'])\n    print(f\"Model and encoder saved: {CONFIG['model_save_path']}, {CONFIG['encoder_save_path']}\")\n\n    # cleanup chunks and processed file (optional)\n    # os.remove(CONFIG['processed_data_path'])\n    # for c in glob.glob(f\"{CONFIG['chunk_prefix']}*.parquet\"): os.remove(c)\n\n    return clf, le, feature_cols\n\n# ---------------------------\n# POSTPROCESS helpers\n# ---------------------------\ndef smooth_preds_by_majority(preds, window=5):\n    \"\"\"\n    preds: numpy array of integer class indices\n    apply majority smoothing with sliding window (odd window recommended)\n    \"\"\"\n    if window <= 1:\n        return preds\n    n = len(preds)\n    half = window // 2\n    smoothed = np.copy(preds)\n    # use deque to keep histogram counts for speed\n    from collections import Counter\n    for i in range(n):\n        a = max(0, i - half)\n        b = min(n, i + half + 1)\n        window_vals = preds[a:b]\n        # majority\n        most = Counter(window_vals).most_common(1)[0][0]\n        smoothed[i] = most\n    return smoothed\n\ndef frames_to_segments(frames, labels, label_map, min_length=3):\n    \"\"\"\n    Convert per-frame label list to list of segments [ (label_name, start, stop) ... ]\n    filters out 'other' and segments shorter than min_length\n    \"\"\"\n    segments = []\n    current_label = labels[0]\n    start = 0\n    for i in range(1, len(labels)):\n        if labels[i] != current_label:\n            segments.append((current_label, start, i-1))\n            current_label = labels[i]\n            start = i\n    segments.append((current_label, start, len(labels)-1))\n    # filter\n    out = []\n    for lab_idx, s, e in segments:\n        lab_name = label_map[lab_idx]\n        if lab_name == 'other':\n            continue\n        if (e - s + 1) < min_length:\n            continue\n        out.append((lab_name, s, e))\n    return out\n\n# ---------------------------\n# INFERENCE & SUBMISSION BUILD\n# ---------------------------\ndef create_submission(clf, le, feature_cols):\n    print(\">>> Phase 3: Inference & Submission Building...\")\n    test_files = sorted(glob.glob(f\"{CONFIG['input_path']}/test_tracking/*/*.parquet\"))\n    submission_rows = []\n    row_id = 0\n\n    for test_path in test_files:\n        video_id = os.path.basename(test_path).replace('.parquet','')\n        try:\n            df_test_trk = pd.read_parquet(test_path)\n            X_test = extract_features_from_tracking(df_test_trk)\n            if X_test is None or X_test.empty:\n                continue\n\n            # align features\n            for c in feature_cols:\n                if c not in X_test.columns:\n                    X_test[c] = 0.0\n            X_test_model = X_test[feature_cols].astype('float32')\n\n            # predict probabilities to allow smoothing-based decisions\n            probs = clf.predict_proba(X_test_model)  # shape (T, n_classes)\n            # Optional: boost 'other' slightly to reduce false positives\n            try:\n                other_idx = list(le.classes_).index('other')\n                probs[:, other_idx] *= 1.05\n            except Exception:\n                pass\n\n            preds = np.argmax(probs, axis=1)\n            # smoothing (majority)\n            preds_sm = smooth_preds_by_majority(preds, window=CONFIG['smoothing_window'])\n\n            # velocity-based filter: if both mice stationary, set to 'other' for movement actions\n            movement_actions = {'chase','escape','follow','attack','approach','chaseattack','attack'}\n            # make bool series: both stationary (speed threshold)\n            if 'm1_speed' in X_test.columns and 'm2_speed' in X_test.columns:\n                is_stationary = (X_test['m1_speed'] < 0.5) & (X_test['m2_speed'] < 0.5)\n            else:\n                is_stationary = pd.Series([False]*len(X_test))\n\n            # map preds_sm -> label names, but first convert to names array\n            label_map = {i:cl for i,cl in enumerate(le.classes_)}\n\n            # apply velocity mask by setting frame preds to index of 'other'\n            try:\n                other_index = int(list(le.classes_).index('other'))\n            except Exception:\n                other_index = None\n\n            if other_index is not None:\n                for i in range(len(preds_sm)):\n                    if is_stationary.iloc[i] and label_map[preds_sm[i]] in movement_actions:\n                        preds_sm[i] = other_index\n\n            # convert frame preds to segments\n            segments = frames_to_segments(X_test[CONFIG['frame_col']].values, preds_sm, label_map,\n                                          min_length=CONFIG['min_segment_frames'])\n\n            # Create rows with agent/target extraction (best-effort)\n            # Attempt to obtain mouse ids from original test file\n            try:\n                if 'mouse_id' in df_test_trk.columns:\n                    u_m = df_test_trk['mouse_id'].unique()\n                    agent_id = str(u_m[0]) if len(u_m)>0 else 'mouse1'\n                    target_id = str(u_m[1]) if len(u_m)>1 else 'mouse2'\n                else:\n                    agent_id, target_id = 'mouse1', 'mouse2'\n            except Exception:\n                agent_id, target_id = 'mouse1', 'mouse2'\n\n            for lab_name, s, e in segments:\n                submission_rows.append([row_id, video_id, agent_id, target_id, lab_name, int(s), int(e)])\n                row_id += 1\n\n            del df_test_trk, X_test, probs, preds, preds_sm\n            gc.collect()\n\n        except Exception as e:\n            # skip problematic test files but keep running\n            # print(\"skip test\", test_path, e)\n            continue\n\n    sub_df = pd.DataFrame(submission_rows, columns=['row_id','video_id','agent_id','target_id','action','start_frame','stop_frame'])\n    if sub_df.empty:\n        # fallback: dummy small submission to avoid empty submission\n        sub_df = pd.DataFrame([[0, 'no_video', 'mouse1', 'mouse2', 'other', 0, 10]],\n                              columns=['row_id','video_id','agent_id','target_id','action','start_frame','stop_frame'])\n    sub_df.to_csv(CONFIG['submission_name'], index=False)\n    print(f\"Saved submission: {CONFIG['submission_name']} (rows={len(sub_df)})\")\n    return sub_df\n\n# ---------------------------\n# MAIN\n# ---------------------------\nif __name__ == \"__main__\":\n    # 1) Process/train only if run as main\n    n_processed = process_and_save_data()\n    if n_processed == 0:\n        print(\"No videos processed. Check CONFIG['input_path'] and that parquet files exist.\")\n    else:\n        clf, le, feature_cols = train_model()\n        sub = create_submission(clf, le, feature_cols)\n        print(\"Pipeline completed.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T09:17:24.322661Z","iopub.execute_input":"2025-12-11T09:17:24.323334Z","iopub.status.idle":"2025-12-11T09:21:03.097828Z","shell.execute_reply.started":"2025-12-11T09:17:24.323310Z","shell.execute_reply":"2025-12-11T09:21:03.097133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_sub = pd.read_csv(\"/kaggle/working/submission.csv\")\ndf_sub","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T09:21:20.809075Z","iopub.execute_input":"2025-12-11T09:21:20.809728Z","iopub.status.idle":"2025-12-11T09:21:20.822735Z","shell.execute_reply.started":"2025-12-11T09:21:20.809698Z","shell.execute_reply":"2025-12-11T09:21:20.821912Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(df_sub.head())\nprint(\"Unique Actions Predicted:\", df_sub['action'].unique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T09:21:22.808026Z","iopub.execute_input":"2025-12-11T09:21:22.808845Z","iopub.status.idle":"2025-12-11T09:21:22.814846Z","shell.execute_reply.started":"2025-12-11T09:21:22.808819Z","shell.execute_reply":"2025-12-11T09:21:22.814207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # =========================================================\n# # RUN THIS IN A SEPARATE CELL TO TEST SAVED MODEL\n# # =========================================================\n# import joblib\n# import pandas as pd\n# import glob\n# import os\n# import numpy as np\n\n# def test_saved_model(test_video_path=None):\n#     print(\">>> Loading Saved Model...\")\n    \n#     # 1. Load Model & Encoder\n#     try:\n#         clf = joblib.load('/kaggle/working/lgbm_model.pkl')\n#         le = joblib.load('/kaggle/working/label_encoder.pkl')\n#         print(\"Model loaded successfully.\")\n#     except FileNotFoundError:\n#         print(\"Error: Model files not found. Run the training pipeline first!\")\n#         return\n\n#     # 2. Pick a random test video if none provided\n#     if test_video_path is None:\n#         possible_files = glob.glob('/kaggle/input/MABe-mouse-behavior-detection/test_tracking/*/*.parquet')\n#         if not possible_files:\n#             print(\"No test files found.\")\n#             return\n#         test_video_path = possible_files[0]\n\n#     print(f\"Testing on video: {os.path.basename(test_video_path)}\")\n    \n#     # 3. Process Data (Must match training processing exactly!)\n#     # We re-define extract_features here briefly or ensure the function above is available\n#     df_test = pd.read_parquet(test_video_path)\n    \n#     # --- Quick Feature Extraction Copy ---\n#     # (Assuming extract_features is defined in the notebook environment)\n#     X_test = extract_features(df_test) \n    \n#     if X_test is None: \n#         print(\"Feature extraction failed.\")\n#         return\n\n#     # Align columns with model\n#     # LightGBM requires exact column match. We get feature names from the model.\n#     model_cols = clf.feature_name_\n#     for col in model_cols:\n#         if col not in X_test.columns:\n#             X_test[col] = 0.0\n    \n#     # Ensure order matches\n#     X_test = X_test[model_cols]\n    \n#     # 4. Predict\n#     preds = clf.predict(X_test)\n#     pred_labels = le.inverse_transform(preds)\n    \n#     # 5. Show Results\n#     print(\"\\nPrediction Summary:\")\n#     unique_actions, counts = np.unique(pred_labels, return_counts=True)\n#     for action, count in zip(unique_actions, counts):\n#         print(f\"  - {action}: {count} frames\")\n        \n#     print(\"\\nFirst 20 Frames:\")\n#     print(pred_labels[:20])\n\n# # Run the test\n# test_saved_model()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-11T09:21:50.009859Z","iopub.execute_input":"2025-12-11T09:21:50.010416Z","iopub.status.idle":"2025-12-11T09:21:50.137981Z","shell.execute_reply.started":"2025-12-11T09:21:50.010392Z","shell.execute_reply":"2025-12-11T09:21:50.136976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}