{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\nimport json\nimport os\nimport warnings\nimport gc\nwarnings.filterwarnings('ignore')\n\n# Machine Learning\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.metrics import f1_score\n\nprint(\"✓ Libraries imported\")\n\n# Configuration\nclass Config:\n    # Paths - ADAPTED TO YOUR DATA\n    DATA_PATH = '/kaggle/input/MABe-mouse-behavior-detection'  # Update this!\n    \n    # Mode\n    MODE = 'submit'  # 'submit'\n    VERBOSE = True\n    \n    # Feature Engineering\n    VELOCITY_WINDOWS = [5, 10, 20]\n    ROLLING_WINDOWS = [10, 20, 30]\n    \n    # Model Parameters\n    THRESHOLD = 0.25\n    DOWNSAMPLE = 2  # Use every Nth frame\n    \n    # LightGBM\n    LGBM_PARAMS = {\n        'objective': 'binary',\n        'metric': 'binary_logloss',\n        'boosting_type': 'gbdt',\n        'num_leaves': 63,\n        'learning_rate': 0.02,\n        'feature_fraction': 0.8,\n        'bagging_fraction': 0.8,\n        'bagging_freq': 5,\n        'min_child_samples': 20,\n        'reg_alpha': 0.1,\n        'reg_lambda': 0.1,\n        'n_estimators': 200,\n        'verbose': -1,\n        'n_jobs': -1,\n        'random_state': 42\n    }\n    \n    # XGBoost\n    XGB_PARAMS = {\n        'objective': 'binary:logistic',\n        'max_depth': 6,\n        'learning_rate': 0.02,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'min_child_weight': 3,\n        'reg_alpha': 0.1,\n        'reg_lambda': 1.0,\n        'n_estimators': 200,\n        'verbosity': 0,\n        'n_jobs': -1,\n        'random_state': 42\n    }\n\nconfig = Config()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:19:54.551215Z","iopub.execute_input":"2025-10-25T06:19:54.551423Z","iopub.status.idle":"2025-10-25T06:20:01.310406Z","shell.execute_reply.started":"2025-10-25T06:19:54.551405Z","shell.execute_reply":"2025-10-25T06:20:01.309723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"=\"*60)\nprint(\"VERIFYING DATA STRUCTURE\")\nprint(\"=\"*60)\n\n# Check data path\nprint(f\"\\nChecking: {config.DATA_PATH}\")\nif os.path.exists(config.DATA_PATH):\n    print(\"✓ Data path found!\")\n    \n    # List contents\n    contents = os.listdir(config.DATA_PATH)\n    print(f\"\\nContents ({len(contents)} items):\")\n    for item in sorted(contents):\n        print(f\"  - {item}\")\n    \n    # Check train_tracking labs\n    train_tracking_path = f\"{config.DATA_PATH}/train_tracking\"\n    if os.path.exists(train_tracking_path):\n        labs = os.listdir(train_tracking_path)\n        print(f\"\\n✓ Found {len(labs)} labs in train_tracking:\")\n        print(f\"  Labs: {labs[:5]}...\")  # Show first 5\n    \n    # Load metadata\n    train = pd.read_csv(f'{config.DATA_PATH}/train.csv')\n    test = pd.read_csv(f'{config.DATA_PATH}/test.csv')\n    \n    print(f\"\\n✓ train.csv loaded: {train.shape}\")\n    print(f\"✓ test.csv loaded: {test.shape}\")\n    \n    print(\"\\nTrain columns:\", train.columns.tolist())\n    \nelse:\n    print(\"❌ Data path not found!\")\n    print(\"Available inputs:\")\n    print(os.listdir('/kaggle/input/'))\n    print(\"\\n⚠️  Please update config.DATA_PATH to match your input folder name\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:20:01.311652Z","iopub.execute_input":"2025-10-25T06:20:01.312193Z","iopub.status.idle":"2025-10-25T06:20:01.430108Z","shell.execute_reply.started":"2025-10-25T06:20:01.312174Z","shell.execute_reply":"2025-10-25T06:20:01.429466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"EXPLORING SAMPLE VIDEO\")\nprint(\"=\"*60)\n\n# Filter out MABe22 (no annotations)\ntrain_clean = train[~train['lab_id'].str.startswith('MABe22')].reset_index(drop=True)\nprint(f\"Train videos (clean): {len(train_clean)}\")\n\n# Load first video\nsample_row = train_clean.iloc[0]\nprint(f\"\\nSample video:\")\nprint(f\"  Video ID: {sample_row['video_id']}\")\nprint(f\"  Lab: {sample_row['lab_id']}\")\nprint(f\"  Behaviors: {sample_row['behaviors_labeled']}\")\n\n# Load tracking data\nlab_id = sample_row['lab_id']\nvideo_id = sample_row['video_id']\ntracking_path = f\"{config.DATA_PATH}/train_tracking/{lab_id}/{video_id}.parquet\"\n\nif os.path.exists(tracking_path):\n    vid = pd.read_parquet(tracking_path)\n    print(f\"\\n✓ Tracking data loaded: {vid.shape}\")\n    print(f\"  Columns: {vid.columns.tolist()}\")\n    print(f\"\\n  First few rows:\")\n    print(vid.head())\n    \n    print(f\"\\n  Unique mice: {vid['mouse_id'].unique()}\")\n    print(f\"  Unique body parts: {vid['bodypart'].unique()}\")\n    print(f\"  Frame range: {vid['video_frame'].min()} to {vid['video_frame'].max()}\")\n    \n    # Check for missing values\n    missing = vid[['x', 'y']].isna().sum()\n    print(f\"\\n  Missing values: {missing.sum()} / {len(vid)*2} ({missing.sum()/(len(vid)*2)*100:.1f}%)\")\nelse:\n    print(f\"❌ Tracking file not found at: {tracking_path}\")\n\n# Load annotation\nannot_path = f\"{config.DATA_PATH}/train_annotation/{lab_id}/{video_id}.parquet\"\nif os.path.exists(annot_path):\n    annot = pd.read_parquet(annot_path)\n    print(f\"\\n✓ Annotation loaded: {annot.shape}\")\n    print(f\"  Columns: {annot.columns.tolist()}\")\n    print(f\"\\n  Sample annotations:\")\n    print(annot.head())\n    print(f\"\\n  Unique actions: {annot['action'].unique()}\")\nelse:\n    print(f\"❌ Annotation not found\")\n\nprint(\"\\n✓ Data exploration complete!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:20:01.430844Z","iopub.execute_input":"2025-10-25T06:20:01.431628Z","iopub.status.idle":"2025-10-25T06:20:01.721870Z","shell.execute_reply.started":"2025-10-25T06:20:01.431583Z","shell.execute_reply":"2025-10-25T06:20:01.721129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"DEFINING FEATURE ENGINEERING\")\nprint(\"=\"*60)\n\nclass MotionFeatures:\n    \"\"\"Extract velocity and acceleration features\"\"\"\n    \n    @staticmethod\n    def compute_velocity(coords, window=5):\n        \"\"\"Compute velocity magnitude\"\"\"\n        velocity = np.zeros(len(coords))\n        if len(coords) > window:\n            velocity[window:] = np.linalg.norm(\n                coords[window:] - coords[:-window], axis=1\n            ) / window\n        return velocity\n    \n    @staticmethod\n    def compute_acceleration(coords, window=5):\n        \"\"\"Compute acceleration magnitude\"\"\"\n        vel = MotionFeatures.compute_velocity(coords, window)\n        accel = np.zeros_like(vel)\n        if len(vel) > window:\n            accel[window:] = (vel[window:] - vel[:-window]) / window\n        return accel\n\nclass GeometricFeatures:\n    \"\"\"Extract geometric features\"\"\"\n    \n    @staticmethod\n    def compute_angle(p1, p2, p3):\n        \"\"\"Compute angle at p2 formed by p1-p2-p3\"\"\"\n        v1 = p1 - p2\n        v2 = p3 - p2\n        \n        v1_norm = np.linalg.norm(v1, axis=1, keepdims=True) + 1e-8\n        v2_norm = np.linalg.norm(v2, axis=1, keepdims=True) + 1e-8\n        \n        v1 = v1 / v1_norm\n        v2 = v2 / v2_norm\n        \n        cos_angle = np.sum(v1 * v2, axis=1)\n        cos_angle = np.clip(cos_angle, -1.0, 1.0)\n        \n        return np.arccos(cos_angle)\n    \n    @staticmethod\n    def compute_distance(coords1, coords2):\n        \"\"\"Euclidean distance\"\"\"\n        return np.sqrt(np.sum((coords1 - coords2)**2, axis=1))\n\ndef extract_single_mouse_features(mouse_df, config):\n    \"\"\"\n    Extract features for single mouse\n    \n    Args:\n        mouse_df: DataFrame with columns (bodypart, xy) for one mouse\n        config: Configuration object\n    \n    Returns:\n        DataFrame with features\n    \"\"\"\n    features = {}\n    body_parts = mouse_df.columns.get_level_values(0).unique().tolist()\n    \n    # 1. POSITION FEATURES (relative to center)\n    if 'center' in body_parts:\n        center = mouse_df['center'].values\n    elif 'spine_middle' in body_parts:\n        center = mouse_df['spine_middle'].values\n    else:\n        # Use mean of all parts\n        center = mouse_df.values.reshape(len(mouse_df), -1, 2).mean(axis=1)\n    \n    for part in body_parts:\n        if part in mouse_df.columns:\n            coords = mouse_df[part].values\n            rel_coords = coords - center\n            features[f'{part}_rel_dist'] = np.linalg.norm(rel_coords, axis=1)\n    \n    # 2. MOTION FEATURES\n    motion = MotionFeatures()\n    for part in body_parts[:3]:  # Use top 3 parts to save memory\n        if part in mouse_df.columns:\n            coords = mouse_df[part].values\n            for w in config.VELOCITY_WINDOWS:\n                features[f'{part}_vel_{w}'] = motion.compute_velocity(coords, w)\n    \n    # 3. GEOMETRIC FEATURES\n    geom = GeometricFeatures()\n    \n    # Body angle (if nose, spine_middle, tail_base exist)\n    if all(p in body_parts for p in ['nose', 'spine_middle', 'tail_base']):\n        angle = geom.compute_angle(\n            mouse_df['nose'].values,\n            mouse_df['spine_middle'].values,\n            mouse_df['tail_base'].values\n        )\n        features['body_angle'] = angle\n    \n    # 4. DISTANCE FEATURES\n    key_parts = [p for p in ['nose', 'spine_middle', 'tail_base'] if p in body_parts]\n    for i, p1 in enumerate(key_parts):\n        for p2 in key_parts[i+1:]:\n            dist = geom.compute_distance(mouse_df[p1].values, mouse_df[p2].values)\n            features[f'dist_{p1}_{p2}'] = dist\n    \n    # Convert to DataFrame\n    features_df = pd.DataFrame(features, index=mouse_df.index)\n    \n    # 5. ROLLING AGGREGATIONS\n    for col in features_df.columns:\n        for w in config.ROLLING_WINDOWS[:2]:  # Use 2 windows to save memory\n            features_df[f'{col}_mean_{w}'] = features_df[col].rolling(\n                window=w, min_periods=1, center=True\n            ).mean()\n    \n    return features_df\n\ndef extract_pair_features(mouse_pair_df, config):\n    \"\"\"\n    Extract features for mouse pair\n    \n    Args:\n        mouse_pair_df: DataFrame with columns (A/B, bodypart, xy)\n        config: Configuration object\n    \n    Returns:\n        DataFrame with features\n    \"\"\"\n    features = {}\n    \n    mouse_A = mouse_pair_df['A']\n    mouse_B = mouse_pair_df['B']\n    \n    parts_A = mouse_A.columns.get_level_values(0).unique().tolist()\n    parts_B = mouse_B.columns.get_level_values(0).unique().tolist()\n    \n    geom = GeometricFeatures()\n    \n    # 1. INTER-MOUSE DISTANCES\n    key_parts = [p for p in ['nose', 'center', 'tail_base'] \n                 if p in parts_A and p in parts_B]\n    \n    for pa in key_parts:\n        for pb in key_parts:\n            dist = geom.compute_distance(mouse_A[pa].values, mouse_B[pb].values)\n            features[f'dist_{pa}_{pb}'] = dist\n    \n    # 2. MINIMUM DISTANCE (closest approach)\n    all_dists = []\n    for pa in parts_A:\n        if pa in mouse_A.columns:\n            for pb in parts_B:\n                if pb in mouse_B.columns:\n                    dist = geom.compute_distance(mouse_A[pa].values, mouse_B[pb].values)\n                    all_dists.append(dist)\n    \n    if all_dists:\n        all_dists = np.array(all_dists).T\n        features['min_dist'] = np.min(all_dists, axis=1)\n        features['mean_dist'] = np.mean(all_dists, axis=1)\n    \n    # 3. RELATIVE MOTION\n    if 'nose' in parts_A and 'nose' in parts_B:\n        motion = MotionFeatures()\n        for w in config.VELOCITY_WINDOWS[:2]:\n            vel_A = motion.compute_velocity(mouse_A['nose'].values, w)\n            vel_B = motion.compute_velocity(mouse_B['nose'].values, w)\n            features[f'vel_diff_{w}'] = np.abs(vel_A - vel_B)\n    \n    # Convert to DataFrame\n    features_df = pd.DataFrame(features, index=mouse_pair_df.index)\n    \n    # 4. ROLLING AGGREGATIONS\n    for col in features_df.columns:\n        for w in config.ROLLING_WINDOWS[:2]:\n            features_df[f'{col}_mean_{w}'] = features_df[col].rolling(\n                window=w, min_periods=1, center=True\n            ).mean()\n    \n    return features_df\n\nprint(\"✓ Feature engineering functions defined\")\nprint(\"  - Motion: velocity, acceleration\")\nprint(\"  - Geometric: angles, distances\")\nprint(\"  - Temporal: rolling aggregations\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:20:01.723693Z","iopub.execute_input":"2025-10-25T06:20:01.723903Z","iopub.status.idle":"2025-10-25T06:20:01.742184Z","shell.execute_reply.started":"2025-10-25T06:20:01.723887Z","shell.execute_reply":"2025-10-25T06:20:01.741450Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"DEFINING VIDEO PROCESSING\")\nprint(\"=\"*60)\n\ndef load_and_pivot_video(lab_id, video_id, traintest='train'):\n    \"\"\"Load video and pivot to wide format\"\"\"\n    path = f\"{config.DATA_PATH}/{traintest}_tracking/{lab_id}/{video_id}.parquet\"\n    \n    try:\n        vid = pd.read_parquet(path)\n        \n        # Pivot: one row per frame\n        pvid = vid.pivot(\n            columns=['mouse_id', 'bodypart'],\n            index='video_frame',\n            values=['x', 'y']\n        )\n        \n        # Reorder to (mouse_id, bodypart, xy)\n        pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T\n        \n        return pvid\n    except Exception as e:\n        if config.VERBOSE:\n            print(f\"Error loading {video_id}: {e}\")\n        return None\n\ndef process_video(row, traintest='train'):\n    \"\"\"\n    Process one video and extract all batches\n    \n    Returns:\n        List of (switch, features, meta, labels/actions)\n    \"\"\"\n    lab_id = row['lab_id']\n    video_id = row['video_id']\n    \n    # Skip MABe22\n    if str(lab_id).startswith('MABe22'):\n        return []\n    \n    # Load video\n    pvid = load_and_pivot_video(lab_id, video_id, traintest)\n    if pvid is None or len(pvid) == 0:\n        return []\n    \n    # Normalize to cm\n    pvid = pvid / row['pix_per_cm_approx']\n    \n    # Parse behaviors\n    try:\n        behaviors = json.loads(row['behaviors_labeled'])\n        behaviors = sorted(list({b.replace(\"'\", \"\") for b in behaviors}))\n        behaviors = [b.split(',') for b in behaviors]\n        behaviors_df = pd.DataFrame(behaviors, columns=['agent', 'target', 'action'])\n    except:\n        return []\n    \n    results = []\n    \n    # ==== SINGLE MOUSE BEHAVIORS ====\n    single_behaviors = behaviors_df[behaviors_df['target'] == 'self']\n    \n    for mouse_str in single_behaviors['agent'].unique():\n        try:\n            mouse_id = int(mouse_str.replace('mouse', ''))\n            \n            if mouse_id not in pvid.columns.get_level_values(0):\n                continue\n            \n            mouse_data = pvid[mouse_id]\n            \n            # Extract features\n            features = extract_single_mouse_features(mouse_data, config)\n            \n            # Downsample for training\n            if traintest == 'train' and config.DOWNSAMPLE > 1:\n                features = features.iloc[::config.DOWNSAMPLE]\n            \n            # Meta\n            meta = pd.DataFrame({\n                'video_id': video_id,\n                'agent_id': mouse_str,\n                'target_id': 'self',\n                'video_frame': features.index\n            })\n            \n            actions = single_behaviors[single_behaviors['agent'] == mouse_str]['action'].unique()\n            \n            if traintest == 'train':\n                # Load annotations\n                annot_path = f\"{config.DATA_PATH}/train_annotation/{lab_id}/{video_id}.parquet\"\n                try:\n                    annot = pd.read_parquet(annot_path)\n                    \n                    # Create labels\n                    labels = pd.DataFrame(0, index=features.index, columns=actions)\n                    \n                    annot_sub = annot[(annot['agent_id'] == mouse_id) & (annot['target_id'] == mouse_id)]\n                    for _, arow in annot_sub.iterrows():\n                        if arow['action'] in actions:\n                            start = arow['start_frame']\n                            stop = arow['stop_frame']\n                            mask = (labels.index >= start) & (labels.index <= stop)\n                            labels.loc[mask, arow['action']] = 1\n                    \n                    results.append(('single', features, meta, labels))\n                except:\n                    pass\n            else:\n                results.append(('single', features, meta, actions))\n        except:\n            pass\n    \n    # ==== MOUSE PAIR BEHAVIORS ====\n    pair_behaviors = behaviors_df[behaviors_df['target'] != 'self']\n    \n    if len(pair_behaviors) > 0:\n        available_mice = pvid.columns.get_level_values(0).unique()\n        \n        for agent_id, target_id in [(a, b) for a in available_mice for b in available_mice if a != b]:\n            agent_str = f'mouse{agent_id}'\n            target_str = f'mouse{target_id}'\n            \n            actions = pair_behaviors[\n                (pair_behaviors['agent'] == agent_str) & \n                (pair_behaviors['target'] == target_str)\n            ]['action'].unique()\n            \n            if len(actions) == 0:\n                continue\n            \n            try:\n                mouse_pair = pd.concat([pvid[agent_id], pvid[target_id]], axis=1, keys=['A', 'B'])\n                \n                # Extract features\n                features = extract_pair_features(mouse_pair, config)\n                \n                # Downsample\n                if traintest == 'train' and config.DOWNSAMPLE > 1:\n                    features = features.iloc[::config.DOWNSAMPLE]\n                \n                # Meta\n                meta = pd.DataFrame({\n                    'video_id': video_id,\n                    'agent_id': agent_str,\n                    'target_id': target_str,\n                    'video_frame': features.index\n                })\n                \n                if traintest == 'train':\n                    annot_path = f\"{config.DATA_PATH}/train_annotation/{lab_id}/{video_id}.parquet\"\n                    try:\n                        annot = pd.read_parquet(annot_path)\n                        \n                        labels = pd.DataFrame(0, index=features.index, columns=actions)\n                        \n                        annot_sub = annot[(annot['agent_id'] == agent_id) & (annot['target_id'] == target_id)]\n                        for _, arow in annot_sub.iterrows():\n                            if arow['action'] in actions:\n                                start = arow['start_frame']\n                                stop = arow['stop_frame']\n                                mask = (labels.index >= start) & (labels.index <= stop)\n                                labels.loc[mask, arow['action']] = 1\n                        \n                        results.append(('pair', features, meta, labels))\n                    except:\n                        pass\n                else:\n                    results.append(('pair', features, meta, actions))\n            except:\n                pass\n    \n    return results\n\nprint(\"✓ Video processing functions defined\")\n\nprint(\"\\n\" + \"=\"*60)\nprint(\"SETUP COMPLETE!\")\nprint(\"=\"*60)\nprint(\"\\nReady to process videos and train models!\")\nprint(\"\\nNext: Run training cells to build the model\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:20:01.742923Z","iopub.execute_input":"2025-10-25T06:20:01.743215Z","iopub.status.idle":"2025-10-25T06:20:01.861001Z","shell.execute_reply.started":"2025-10-25T06:20:01.743199Z","shell.execute_reply":"2025-10-25T06:20:01.860079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# **MABe Challenge - Training & Prediction**","metadata":{}},{"cell_type":"code","source":"print(\"=\"*60)\nprint(\"PROCESSING TRAINING DATA\")\nprint(\"=\"*60)\n\n# Group by body parts tracked\nbody_parts_groups = train_clean['body_parts_tracked'].unique()\nprint(f\"Found {len(body_parts_groups)} body part configurations\")\n\n# Storage for training data\ntrain_data = {}\n\nfor bp_idx, bp_str in enumerate(body_parts_groups):\n    print(f\"\\n[{bp_idx+1}/{len(body_parts_groups)}] Processing: {bp_str[:60]}...\")\n    \n    subset = train_clean[train_clean['body_parts_tracked'] == bp_str]\n    print(f\"  Videos: {len(subset)}\")\n    \n    single_data = {'features': [], 'meta': [], 'labels': []}\n    pair_data = {'features': [], 'meta': [], 'labels': []}\n    \n    for _, row in tqdm(subset.iterrows(), total=len(subset), desc=\"  Processing\", leave=False):\n        results = process_video(row, 'train')\n        \n        for switch, feats, meta, labels in results:\n            if switch == 'single':\n                single_data['features'].append(feats)\n                single_data['meta'].append(meta)\n                single_data['labels'].append(labels)\n            else:\n                pair_data['features'].append(feats)\n                pair_data['meta'].append(meta)\n                pair_data['labels'].append(labels)\n    \n    train_data[bp_str] = {\n        'single': single_data,\n        'pair': pair_data\n    }\n    \n    print(f\"  ✓ Single batches: {len(single_data['features'])}\")\n    print(f\"  ✓ Pair batches: {len(pair_data['features'])}\")\n    \n    # Free memory\n    gc.collect()\n\nprint(f\"\\n✓ All training data processed!\")\nprint(f\"Total body part groups: {len(train_data)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:20:01.861744Z","iopub.execute_input":"2025-10-25T06:20:01.861975Z","iopub.status.idle":"2025-10-25T06:23:48.692785Z","shell.execute_reply.started":"2025-10-25T06:20:01.861958Z","shell.execute_reply":"2025-10-25T06:23:48.691976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"TRAINING MODELS\")\nprint(\"=\"*60)\n\ntrained_models = {}\n\nfor bp_idx, (bp_str, data) in enumerate(train_data.items()):\n    print(f\"\\n[{bp_idx+1}/{len(train_data)}] Training for: {bp_str[:60]}...\")\n    \n    # ==== TRAIN SINGLE MOUSE MODELS ====\n    if len(data['single']['features']) > 0:\n        print(\"\\n  [SINGLE MOUSE]\")\n        \n        # Concatenate\n        X = pd.concat(data['single']['features'], ignore_index=True)\n        meta = pd.concat(data['single']['meta'], ignore_index=True)\n        labels = pd.concat(data['single']['labels'], ignore_index=True)\n        \n        print(f\"    Shape: {X.shape}\")\n        print(f\"    Actions: {list(labels.columns)}\")\n        \n        single_models = {}\n        \n        for action in labels.columns:\n            mask = ~labels[action].isna()\n            \n            if mask.sum() < 50:\n                continue\n            \n            X_action = X[mask].fillna(0).values\n            y_action = labels[action][mask].astype(int).values\n            \n            pos_rate = y_action.mean()\n            \n            # Train LightGBM\n            model_lgbm = lgb.LGBMClassifier(**config.LGBM_PARAMS)\n            model_lgbm.fit(X_action, y_action)\n            \n            # Train XGBoost\n            model_xgb = xgb.XGBClassifier(**config.XGB_PARAMS)\n            model_xgb.fit(X_action, y_action)\n            \n            single_models[action] = {\n                'lgbm': model_lgbm,\n                'xgb': model_xgb\n            }\n            \n            print(f\"      ✓ {action}: {mask.sum():,} samples, {pos_rate:.3f} pos\")\n        \n        trained_models[f'{bp_str}__single'] = single_models\n        \n        # Free memory\n        del X, meta, labels\n        gc.collect()\n    \n    # ==== TRAIN PAIR MODELS ====\n    if len(data['pair']['features']) > 0:\n        print(\"\\n  [MOUSE PAIRS]\")\n        \n        # Concatenate\n        X = pd.concat(data['pair']['features'], ignore_index=True)\n        meta = pd.concat(data['pair']['meta'], ignore_index=True)\n        labels = pd.concat(data['pair']['labels'], ignore_index=True)\n        \n        print(f\"    Shape: {X.shape}\")\n        print(f\"    Actions: {list(labels.columns)}\")\n        \n        pair_models = {}\n        \n        for action in labels.columns:\n            mask = ~labels[action].isna()\n            \n            if mask.sum() < 50:\n                continue\n            \n            X_action = X[mask].fillna(0).values\n            y_action = labels[action][mask].astype(int).values\n            \n            pos_rate = y_action.mean()\n            \n            # Train LightGBM\n            model_lgbm = lgb.LGBMClassifier(**config.LGBM_PARAMS)\n            model_lgbm.fit(X_action, y_action)\n            \n            # Train XGBoost\n            model_xgb = xgb.XGBClassifier(**config.XGB_PARAMS)\n            model_xgb.fit(X_action, y_action)\n            \n            pair_models[action] = {\n                'lgbm': model_lgbm,\n                'xgb': model_xgb\n            }\n            \n            print(f\"      ✓ {action}: {mask.sum():,} samples, {pos_rate:.3f} pos\")\n        \n        trained_models[f'{bp_str}__pair'] = pair_models\n        \n        # Free memory\n        del X, meta, labels\n        gc.collect()\n\nprint(f\"\\n{'='*60}\")\nprint(f\"TRAINING COMPLETE!\")\nprint(f\"{'='*60}\")\nprint(f\"Total model groups: {len(trained_models)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:23:48.693704Z","iopub.execute_input":"2025-10-25T06:23:48.693938Z","iopub.status.idle":"2025-10-25T06:53:21.308018Z","shell.execute_reply.started":"2025-10-25T06:23:48.693920Z","shell.execute_reply":"2025-10-25T06:53:21.307359Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_multiclass(preds_df, meta_df, threshold=0.25):\n    \"\"\"Convert probabilities to submission format\"\"\"\n    if len(preds_df.columns) == 0:\n        return pd.DataFrame(columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n    \n    # Get most probable action per frame\n    max_prob = preds_df.max(axis=1).values\n    max_idx = preds_df.values.argmax(axis=1)\n    \n    # Apply threshold\n    action_idx = np.where(max_prob >= threshold, max_idx, -1)\n    action_series = pd.Series(action_idx, index=meta_df['video_frame'].values)\n    \n    # Find changes\n    changes = (action_series != action_series.shift(1))\n    change_frames = action_series[changes]\n    change_meta = meta_df[changes.values].reset_index(drop=True)\n    \n    # Build submission\n    submissions = []\n    \n    for i in range(len(change_frames) - 1):\n        action_idx = change_frames.iloc[i]\n        \n        if action_idx >= 0:  # Valid action\n            video_id = change_meta.iloc[i]['video_id']\n            agent_id = change_meta.iloc[i]['agent_id']\n            target_id = change_meta.iloc[i]['target_id']\n            action = preds_df.columns[action_idx]\n            start_frame = change_frames.index[i]\n            \n            # Check if next change is same video/agent/target\n            next_video = change_meta.iloc[i + 1]['video_id']\n            next_agent = change_meta.iloc[i + 1]['agent_id']\n            next_target = change_meta.iloc[i + 1]['target_id']\n            \n            if video_id == next_video and agent_id == next_agent and target_id == next_target:\n                stop_frame = change_frames.index[i + 1]\n            else:\n                # End of video/agent/target\n                stop_frame = meta_df[meta_df['video_id'] == video_id]['video_frame'].max() + 1\n            \n            submissions.append({\n                'video_id': video_id,\n                'agent_id': agent_id,\n                'target_id': target_id,\n                'action': action,\n                'start_frame': start_frame,\n                'stop_frame': stop_frame\n            })\n    \n    return pd.DataFrame(submissions)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:53:21.308894Z","iopub.execute_input":"2025-10-25T06:53:21.309157Z","iopub.status.idle":"2025-10-25T06:53:21.317090Z","shell.execute_reply.started":"2025-10-25T06:53:21.309128Z","shell.execute_reply":"2025-10-25T06:53:21.316224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nif config.MODE == 'submit':\n    print(\"PROCESSING TEST DATA & MAKING PREDICTIONS\")\n    dataset = test\nelse:\n    print(\"VALIDATION MODE - Using last 20% of train\")\n    val_split = int(len(train_clean) * 0.8)\n    dataset = train_clean.iloc[val_split:]\n    print(f\"Validation videos: {len(dataset)}\")\n\nprint(\"=\"*60)\n\nall_predictions = []\n\n# Process by body parts\nbody_parts_groups = dataset['body_parts_tracked'].unique()\n\nfor bp_idx, bp_str in enumerate(body_parts_groups):\n    print(f\"\\n[{bp_idx+1}/{len(body_parts_groups)}] Predicting: {bp_str[:60]}...\")\n    \n    subset = dataset[dataset['body_parts_tracked'] == bp_str]\n    print(f\"  Videos: {len(subset)}\")\n    \n    # Check if we have models for this body part configuration\n    single_model_key = f'{bp_str}__single'\n    pair_model_key = f'{bp_str}__pair'\n    \n    has_single = single_model_key in trained_models\n    has_pair = pair_model_key in trained_models\n    \n    if not has_single and not has_pair:\n        print(f\"  ⚠️  No trained models for this configuration\")\n        continue\n    \n    # Process each video\n    for _, row in tqdm(subset.iterrows(), total=len(subset), desc=\"  Videos\", leave=False):\n        \n        if config.MODE == 'submit':\n            results = process_video(row, 'test')\n        else:\n            results = process_video(row, 'train')\n        \n        for switch, features, meta, actions_or_labels in results:\n            \n            # Get actions to predict\n            if config.MODE == 'submit':\n                actions = actions_or_labels\n            else:\n                actions = actions_or_labels.columns.tolist()\n            \n            # Select models\n            if switch == 'single' and has_single:\n                models = trained_models[single_model_key]\n            elif switch == 'pair' and has_pair:\n                models = trained_models[pair_model_key]\n            else:\n                continue\n            \n            # Make predictions\n            predictions = pd.DataFrame(index=features.index)\n            \n            for action in actions:\n                if action in models:\n                    # Ensemble: 50% LightGBM + 50% XGBoost\n                    X = features.fillna(0).values\n                    \n                    try:\n                        pred_lgbm = models[action]['lgbm'].predict_proba(X)[:, 1]\n                    except:\n                        pred_lgbm = np.zeros(len(X))\n                    \n                    try:\n                        pred_xgb = models[action]['xgb'].predict_proba(X)[:, 1]\n                    except:\n                        pred_xgb = np.zeros(len(X))\n                    \n                    predictions[action] = 0.5 * pred_lgbm + 0.5 * pred_xgb\n            \n            # Convert to submission format\n            if len(predictions.columns) > 0:\n                submission_part = predict_multiclass(predictions, meta, config.THRESHOLD)\n                all_predictions.append(submission_part)\n    \n    print(f\"  ✓ Predictions: {sum(len(p) for p in all_predictions)}\")\n\n# Combine all predictions\nif len(all_predictions) > 0:\n    submission = pd.concat(all_predictions, ignore_index=True)\nelse:\n    print(\"\\n⚠️  No predictions generated!\")\n    submission = pd.DataFrame(columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n\nprint(f\"\\n✓ Total predictions: {len(submission)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:53:21.318053Z","iopub.execute_input":"2025-10-25T06:53:21.318410Z","iopub.status.idle":"2025-10-25T06:53:30.331262Z","shell.execute_reply.started":"2025-10-25T06:53:21.318386Z","shell.execute_reply":"2025-10-25T06:53:30.330424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"POST-PROCESSING SUBMISSION\")\nprint(\"=\"*60)\n\n# Rule 1: Remove invalid predictions (start >= stop)\noriginal_len = len(submission)\nsubmission = submission[submission['start_frame'] < submission['stop_frame']]\nif len(submission) < original_len:\n    print(f\"  Removed {original_len - len(submission)} invalid predictions\")\n\n# Rule 2: Remove overlapping predictions for same agent/target\ncleaned = []\nfor key, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n    group = group.sort_values('start_frame').reset_index(drop=True)\n    \n    keep = []\n    last_stop = -1\n    \n    for idx, row in group.iterrows():\n        if row['start_frame'] >= last_stop:\n            keep.append(idx)\n            last_stop = row['stop_frame']\n    \n    cleaned.append(group.loc[keep])\n\nsubmission = pd.concat(cleaned, ignore_index=True)\n\nprint(f\"✓ After deduplication: {len(submission)} predictions\")\n\n# Rule 3: Fill missing videos\npredicted_videos = set(submission['video_id'].unique())\nall_videos = set(dataset['video_id'].unique())\nmissing_videos = all_videos - predicted_videos\n\nif len(missing_videos) > 0:\n    print(f\"⚠️  Filling {len(missing_videos)} videos with no predictions\")\n    \n    dummy_preds = []\n    for video_id in missing_videos:\n        row = dataset[dataset['video_id'] == video_id].iloc[0]\n        \n        try:\n            behaviors = json.loads(row['behaviors_labeled'])\n            if len(behaviors) > 0:\n                first_behavior = behaviors[0].replace(\"'\", \"\").split(',')\n                dummy_preds.append({\n                    'video_id': video_id,\n                    'agent_id': first_behavior[0],\n                    'target_id': first_behavior[1],\n                    'action': first_behavior[2],\n                    'start_frame': 0,\n                    'stop_frame': 100\n                })\n        except:\n            pass\n    \n    if dummy_preds:\n        dummy_df = pd.DataFrame(dummy_preds)\n        submission = pd.concat([submission, dummy_df], ignore_index=True)\n\nprint(f\"✓ Final submission: {len(submission)} predictions\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:53:30.333221Z","iopub.execute_input":"2025-10-25T06:53:30.333445Z","iopub.status.idle":"2025-10-25T06:53:30.379477Z","shell.execute_reply.started":"2025-10-25T06:53:30.333429Z","shell.execute_reply":"2025-10-25T06:53:30.378758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"SAVING SUBMISSION\")\nprint(\"=\"*60)\n\n# Add row_id and save\nsubmission = submission.reset_index(drop=True)\nsubmission.index.name = 'row_id'\n\nsubmission.to_csv('submission.csv')\n\nprint(\"✓ Saved to submission.csv\")\nprint(f\"\\nSubmission shape: {submission.shape}\")\nprint(f\"\\nFirst 10 rows:\")\nprint(submission.head(10))\n\nprint(f\"\\nAction distribution:\")\naction_counts = submission['action'].value_counts()\nfor action, count in action_counts.items():\n    print(f\"  {action}: {count}\")\n\nprint(f\"\\nVideos covered: {submission['video_id'].nunique()} / {len(dataset)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T06:53:30.380099Z","iopub.execute_input":"2025-10-25T06:53:30.380279Z","iopub.status.idle":"2025-10-25T06:53:30.396652Z","shell.execute_reply.started":"2025-10-25T06:53:30.380265Z","shell.execute_reply":"2025-10-25T06:53:30.395878Z"}},"outputs":[],"execution_count":null}]}