{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nx = []\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        x.append(str(os.path.join(dirname)))\n\nprint(set(x))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 1: Imports and Setup ---\nimport os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pathlib import Path\n\n# Set a style for plots\nsns.set_style(\"whitegrid\")\nplt.rcParams['figure.figsize'] = (12, 6)\n\n# Define the base input path\nBASE_PATH = Path('/kaggle/input/MABe-mouse-behavior-detection/')","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 2: Load and Inspect Metadata ---\ntrain_meta_df = pd.read_csv(BASE_PATH / 'train.csv')\n\nprint(\"Shape of the training metadata:\")\nprint(train_meta_df.shape)\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(train_meta_df.head())\n\nprint(\"\\nData types and missing values:\")\ntrain_meta_df.info()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 3: Analyze Lab and Behavior Diversity ---\nprint(\"--- Lab Distribution ---\")\nlab_counts = train_meta_df['lab_id'].value_counts()\nprint(lab_counts)\n\nplt.figure(figsize=(12, 7))\nsns.barplot(x=lab_counts.index, y=lab_counts.values, palette='viridis')\nplt.title('Number of Videos per Lab')\nplt.xlabel('Lab ID')\nplt.ylabel('Video Count')\nplt.xticks(rotation=45, ha='right')\nplt.show()\n\nprint(\"\\n--- Unique sets of Body Parts Tracked ---\")\nprint(train_meta_df['body_parts_tracked'].value_counts())\n\nprint(\"\\n--- Tracking Method Distribution ---\")\nprint(train_meta_df['tracking_method'].value_counts())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 4 (Corrected): Load Annotations and Add Metadata ---\nimport re\n\nannotation_files = list(BASE_PATH.glob('train_annotation/*/*.parquet'))\n\nall_annotations_list = []\nfor f in annotation_files:\n    # Extract lab_id and video_id from the file path\n    # The path is like: .../train_annotation/{lab_id}/{video_id}.parquet\n    lab_id = f.parts[-2]\n    video_id = int(f.stem) # f.stem gets the filename without extension\n\n    # Load the data\n    ann_df = pd.read_parquet(f)\n\n    # Add the new columns\n    ann_df['lab_id'] = lab_id\n    ann_df['video_id'] = video_id\n\n    all_annotations_list.append(ann_df)\n\nall_annotations_df = pd.concat(all_annotations_list)\n\nprint(\"--- Annotations DataFrame with video_id and lab_id ---\")\nprint(f\"Total number of annotated events: {len(all_annotations_df)}\")\ndisplay(all_annotations_df.head())\n\n# The rest of the cell (plotting action counts) can remain the same.\naction_counts = all_annotations_df['action'].value_counts()\nprint(\"\\n--- Top 15 Most Frequent Behaviors ---\")\nprint(action_counts.head(15))\n# ... (plotting code) ...\n\nplt.figure(figsize=(14, 8))\n# Let's plot the top 20 for a better view\ntop_n = 20\nsns.barplot(y=action_counts.index[:top_n], x=action_counts.values[:top_n], orient='h', palette='rocket')\nplt.title(f'Top {top_n} Behavior Frequencies Across All Labs')\nplt.xlabel('Number of Events')\nplt.ylabel('Behavior')\nplt.gca().invert_yaxis() # To show the most frequent at the top\nplt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 5 (No changes needed now): Inspect a Single Tracking File ---\n# This cell should now work without errors.\n# Let's find a video that has annotations\nannotated_video_ids = all_annotations_df['video_id'].unique()\nsample_video_id = annotated_video_ids[0]\n\n# We already have the lab_id in our merged annotations dataframe\nsample_lab_id = all_annotations_df[all_annotations_df['video_id'] == sample_video_id].iloc[0]['lab_id']\n\ntracking_file_path = BASE_PATH / f'train_tracking/{sample_lab_id}/{sample_video_id}.parquet'\n\nsample_tracking_df = pd.read_parquet(tracking_file_path)\n\nprint(f\"--- Exploring Video ID: {sample_video_id} from Lab: {sample_lab_id} ---\")\nprint(\"Shape of tracking data:\", sample_tracking_df.shape)\nprint(\"\\nUnique mice:\", sample_tracking_df['mouse_id'].unique())\nprint(\"Unique body parts:\", sample_tracking_df['bodypart'].unique())\nprint(\"\\nFirst 5 rows:\")\ndisplay(sample_tracking_df.head())\n\n# --- Cell 6 (No changes needed): Pivot Tracking Data ---\n# This cell should also work now.\n# ... (pivot function and call) ...","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Cell 6: Pivot Tracking Data to Wide Format ---\n\ndef pivot_tracking_data(df):\n    \"\"\"Pivots the long-format tracking data to a wide format for a single video.\"\"\"\n    # First, get body parts on columns for each mouse\n    pivoted_df = df.pivot_table(\n        index=['video_frame', 'mouse_id'],\n        columns='bodypart',\n        values=['x', 'y']\n    )\n    # Flatten the multi-level column index (e.g., from ('x', 'nose') to 'x_nose')\n    pivoted_df.columns = ['_'.join(col).strip() for col in pivoted_df.columns.values]\n    pivoted_df = pivoted_df.reset_index()\n\n    # Now, get each mouse's data onto the same row for each frame\n    # We need to handle cases with more than 2 mice, but for now, let's focus on mouse1/mouse2\n    # This assumes mouse_ids are like 'mouse1', 'mouse2'\n    if all(pivoted_df['mouse_id'].isin(['mouse1', 'mouse2'])):\n        final_df = pivoted_df.pivot(\n            index='video_frame',\n            columns='mouse_id',\n        )\n        # Flatten the multi-level columns again (e.g., from ('x_nose', 'mouse1') to 'x_nose_mouse1')\n        final_df.columns = ['_'.join(col).strip() for col in final_df.columns.values]\n        final_df = final_df.reset_index()\n    else:\n        # A more general approach if mouse_ids are not standard\n        # This is more complex and we can defer it if not needed for the baseline\n        print(\"Non-standard mouse IDs found. Returning intermediate pivot.\")\n        return pivoted_df\n\n    return final_df\n\nwide_df = pivot_tracking_data(sample_tracking_df)\n\nprint(\"--- Pivoted Wide Format ---\")\nprint(\"Shape of wide data:\", wide_df.shape)\ndisplay(wide_df.head())","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport lightgbm as lgb\nfrom tqdm import tqdm\nimport gc\n\n# --- 1. Configuration ---\nclass CONFIG:\n    BASE_PATH = Path('/kaggle/input/MABe-mouse-behavior-detection/')\n    BEHAVIORS_TO_TRAIN = ['sniff', 'attack', 'rear', 'approach', 'selfgroom']\n    LGB_PARAMS = {\n        'objective': 'binary',\n        'metric': 'binary_logloss',\n        'boosting_type': 'gbdt',\n        'n_estimators': 250,\n        'learning_rate': 0.05,\n        'num_leaves': 20,\n        'max_depth': 5,\n        'seed': 42,\n        'n_jobs': -1,\n        'verbose': -1\n    }\n    PROB_THRESHOLD = 0.5\n    MIN_FRAMES_FOR_EVENT = 3\n    NEGATIVE_SAMPLING_RATIO = 4\n\n# --- 2. Helper Function: Load Annotations ---\ndef load_all_annotations(base_path):\n    print(\"Loading all annotations...\")\n    annotation_files = list(base_path.glob('train_annotation/*/*.parquet'))\n    all_annotations_list = []\n    for f in tqdm(annotation_files):\n        lab_id = f.parts[-2]\n        video_id = int(f.stem)\n        ann_df = pd.read_parquet(f)\n        ann_df['lab_id'] = lab_id\n        ann_df['video_id'] = video_id\n        all_annotations_list.append(ann_df)\n    return pd.concat(all_annotations_list).reset_index(drop=True)\n\n# --- 3. Helper Function: Lightweight Feature Engineering ---\nCANONICAL_SKELETON = ['nose', 'ear_left', 'ear_right', 'neck', 'body_center', 'tail_base']\n\ndef process_video_data_lightweight(video_id, lab_id, tracking_path):\n    tracking_df = pd.read_parquet(tracking_path / f\"{lab_id}/{video_id}.parquet\")\n    tracking_df['mouse_id'] = 'mouse' + tracking_df['mouse_id'].astype(str)\n    cols_to_interpolate = ['x', 'y']\n    tracking_df[cols_to_interpolate] = tracking_df.groupby(['mouse_id', 'bodypart'])[cols_to_interpolate].transform(\n        lambda s: s.interpolate(method='linear', limit_direction='both')\n    )\n    wide_df = tracking_df.pivot_table(index='video_frame', columns=['mouse_id', 'bodypart'], values=['x', 'y']).fillna(0)\n    wide_df.columns = ['_'.join(col) for col in wide_df.columns.values]\n    features_list = []\n    for mouse in ['mouse1', 'mouse2']:\n        part_map = {'nose': 'nose', 'ear_left': 'ear_left', 'ear_right': 'ear_right', 'neck': 'neck', 'body_center': 'body_center', 'tail_base': 'tail_base'}\n        raw_coords = {}\n        for canonical_part in CANONICAL_SKELETON:\n            for potential_name, assigned_name in part_map.items():\n                if assigned_name == canonical_part and f'x_{mouse}_{potential_name}' in wide_df.columns:\n                    raw_coords[f'{mouse}_{canonical_part}_x'] = wide_df[f'x_{mouse}_{potential_name}']\n                    raw_coords[f'{mouse}_{canonical_part}_y'] = wide_df[f'y_{mouse}_{potential_name}']\n                    break\n            if f'{mouse}_{canonical_part}_x' not in raw_coords:\n                 raw_coords[f'{mouse}_{canonical_part}_x'] = 0\n                 raw_coords[f'{mouse}_{canonical_part}_y'] = 0\n        center_x, center_y = raw_coords.get(f'{mouse}_body_center_x', 0), raw_coords.get(f'{mouse}_body_center_y', 0)\n        for part in CANONICAL_SKELETON:\n            x_norm = pd.Series(raw_coords.get(f'{mouse}_{part}_x', 0) - center_x, name=f'{mouse}_{part}_x_norm')\n            y_norm = pd.Series(raw_coords.get(f'{mouse}_{part}_y', 0) - center_y, name=f'{mouse}_{part}_y_norm')\n            features_list.extend([x_norm, y_norm])\n    social_features_defs = [('nose', 'nose'), ('nose', 'body_center'), ('nose', 'tail_base'), ('body_center', 'body_center')]\n    for part1, part2 in social_features_defs:\n        try:\n            m1_x, m1_y = wide_df[f'x_mouse1_{part1}'], wide_df[f'y_mouse1_{part1}']\n            m2_x, m2_y = wide_df[f'x_mouse2_{part2}'], wide_df[f'y_mouse2_{part2}']\n            dist = np.sqrt((m1_x - m2_x)**2 + (m1_y - m2_y)**2)\n            dist_series = dist.fillna(0); dist_series.name = f'dist_m1{part1}_m2{part2}'\n            features_list.append(dist_series)\n        except KeyError: pass\n    return pd.concat(features_list, axis=1).reset_index(drop=True)\n\n# --- 4. Helper Function: Post-Processing [THIS FUNCTION WAS MISSING] ---\ndef probabilities_to_events(probs, threshold, min_frames):\n    if probs is None or len(probs) == 0: return []\n    binary_preds = (probs > threshold).astype(int)\n    diffs = np.diff(binary_preds, prepend=0, append=0)\n    starts = np.where(diffs == 1)[0]\n    stops = np.where(diffs == -1)[0]\n    events = []\n    for start, stop in zip(starts, stops):\n        if stop - start >= min_frames:\n            events.append((start, stop - 1))\n    return events\n\n# --- 5. Main Training and Inference Logic ---\nif __name__ == \"__main__\":\n    train_meta_df = pd.read_csv(CONFIG.BASE_PATH / 'train.csv')\n    test_meta_df = pd.read_csv(CONFIG.BASE_PATH / 'test.csv')\n    \n    all_annotations_df = load_all_annotations(CONFIG.BASE_PATH)\n    annotated_video_ids = all_annotations_df['video_id'].unique()\n    train_meta_df_filtered = train_meta_df[train_meta_df['video_id'].isin(annotated_video_ids)].copy()\n    \n    models = {}\n\n    for behavior in CONFIG.BEHAVIORS_TO_TRAIN:\n        print(f\"\\n--- Processing & Sampling data for behavior: {behavior} ---\")\n        X_train_list, y_train_list = [], []\n        for row in tqdm(train_meta_df_filtered.itertuples(), total=len(train_meta_df_filtered), desc=f\"Videos for {behavior}\"):\n            try:\n                features_df = process_video_data_lightweight(row.video_id, row.lab_id, CONFIG.BASE_PATH / 'train_tracking')\n                if features_df.empty: continue\n            except (FileNotFoundError, KeyError) as e: continue\n            labels = pd.Series(np.zeros(len(features_df)), name=behavior)\n            video_ann = all_annotations_df[(all_annotations_df['video_id'] == row.video_id) & (all_annotations_df['action'] == behavior)]\n            for _, ann_row in video_ann.iterrows():\n                start, stop = ann_row['start_frame'], ann_row['stop_frame']\n                if start < len(labels): labels.iloc[start : stop + 1] = 1\n            positive_indices, negative_indices = labels[labels == 1].index, labels[labels == 0].index\n            if len(positive_indices) == 0: continue\n            num_neg_to_sample = min(int(len(positive_indices) * CONFIG.NEGATIVE_SAMPLING_RATIO), len(negative_indices))\n            sampled_negative_indices = np.random.choice(negative_indices, size=num_neg_to_sample, replace=False)\n            final_indices = np.concatenate([positive_indices, sampled_negative_indices])\n            X_train_list.append(features_df.iloc[final_indices]); y_train_list.append(labels.iloc[final_indices])\n        if not X_train_list: print(f\"No data for {behavior} after sampling. Skipping.\"); continue\n        X_train, y_train = pd.concat(X_train_list).reset_index(drop=True), pd.concat(y_train_list).reset_index(drop=True)\n        print(f\"--- Training model for: {behavior} ---\"); print(f\"Sampled training data shape: X={X_train.shape}, y={y_train.shape}\")\n        model = lgb.LGBMClassifier(**CONFIG.LGB_PARAMS); model.fit(X_train, y_train); models[behavior] = model\n        del X_train, y_train, X_train_list, y_train_list; gc.collect()\n\n    print(\"\\n--- Starting Inference on Test Set ---\")\n    all_predictions = []\n    for row in tqdm(test_meta_df.itertuples(), total=len(test_meta_df), desc=\"Inferring on test videos\"):\n        try:\n            features_df = process_video_data_lightweight(row.video_id, row.lab_id, CONFIG.BASE_PATH / 'test_tracking')\n            if features_df.empty: continue\n        except (FileNotFoundError, KeyError): continue\n        for behavior, model in models.items():\n            probs = model.predict_proba(features_df)[:, 1]\n            events = probabilities_to_events(probs, CONFIG.PROB_THRESHOLD, CONFIG.MIN_FRAMES_FOR_EVENT)\n            for start, stop in events:\n                agent_id = 'mouse1'\n                target_id = 'mouse2' if behavior not in ['selfgroom', 'rear'] else 'mouse1'\n                all_predictions.append({'video_id': row.video_id, 'agent_id': agent_id, 'target_id': target_id, 'action': behavior, 'start_frame': start, 'stop_frame': stop})\n    \n    submission_df = pd.DataFrame(all_predictions)\n    if not submission_df.empty:\n        submission_df['row_id'] = submission_df.index\n        submission_df = submission_df[['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']]\n    else:\n        submission_df = pd.DataFrame(columns=['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n    submission_df.to_csv('submission.csv', index=False)\n    print(\"\\nSubmission file 'submission.csv' created successfully.\")\n    print(submission_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-14T18:32:16.003122Z","iopub.execute_input":"2025-10-14T18:32:16.003426Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}