{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# In[1]:\n# ===================================================================\n# Section 1: Setup, Configuration, and Helper Classes\n# ===================================================================\nimport pandas as pd\nimport numpy as np\nfrom tqdm.notebook import tqdm\nimport itertools\nimport warnings\nimport json\nimport os\nimport random\nimport gc\nfrom collections import defaultdict\n\n# Import Models\nimport lightgbm as lgb\nfrom xgboost import XGBClassifier\n\n# Sklearn utilities\nfrom sklearn.base import ClassifierMixin, BaseEstimator, clone\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.model_selection import StratifiedShuffleSplit\nfrom sklearn.impute import SimpleImputer\n\n# Suppress warnings for cleaner output\nwarnings.filterwarnings('ignore')\n\n# --- Configuration Class ---\nclass CFG:\n    \"\"\"\n    Configuration class for all hyperparameters and settings.\n    Includes a DEBUG toggle for fast pipeline testing.\n    \"\"\"\n    # Set DEBUG to True to run on a 5% sample of the data for quick testing.\n    # Set to False for a full training run.\n    DEBUG = False\n    \n    # Paths\n    BASE_PATH = \"/kaggle/input/MABe-mouse-behavior-detection\"\n    \n    # Training parameters\n    N_SAMPLES_SINGLE = 2_000_000  # Number of samples for training single-mouse models\n    N_SAMPLES_PAIR = 900_000   # Number of samples for training mouse-pair models\n    \n    # Prediction parameters\n    PREDICTION_THRESHOLD = 0.27\n    # Action-specific thresholds can improve F1 score.\n    # These are heuristics derived from public notebooks.\n    ACTION_THRESHOLDS = {\n        \"default\": 0.27,\n        \"single_default\": 0.27,\n        \"pair_default\": 0.27,\n        \"single\": {\"rear\": 0.30},\n    }\n\n    # Seed for reproducibility\n    SEED = 42\n\n# --- Seed for Reproducibility ---\ndef seed_everything(seed):\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\n\nseed_everything(CFG.SEED)\n\n# --- Helper Classifier for Memory Management ---\nclass StratifiedSubsetClassifier(ClassifierMixin, BaseEstimator):\n    \"\"\"\n    A wrapper for scikit-learn estimators that fits the model on a stratified\n    random subset of the training data. This is crucial for handling datasets\n    that don't fit into memory.\n    \"\"\"\n    def __init__(self, estimator, n_samples=None):\n        self.estimator = estimator\n        self.n_samples = n_samples\n\n    def _to_numpy(self, X):\n        if hasattr(X, 'to_numpy'):\n            return X.to_numpy(np.float32, copy=False)\n        return np.asarray(X, dtype=np.float32)\n\n    def fit(self, X, y):\n        Xn = self._to_numpy(X)\n        y = np.asarray(y).ravel()\n\n        if self.n_samples is None or len(Xn) <= int(self.n_samples):\n            self.estimator.fit(Xn, y)\n        else:\n            sss = StratifiedShuffleSplit(n_splits=1, train_size=int(self.n_samples), random_state=CFG.SEED)\n            try:\n                # Use stratified sampling to maintain class distribution\n                idx, _ = next(sss.split(np.zeros_like(y), y))\n                self.estimator.fit(Xn[idx], y[idx])\n            except Exception:\n                # Fallback to simple downsampling if stratification fails (e.g., too few samples of a class)\n                step = max(len(Xn) // int(self.n_samples), 1)\n                self.estimator.fit(Xn[::step], y[::step])\n        \n        self.classes_ = self.estimator.classes_\n        return self\n\n    def predict_proba(self, X):\n        Xn = self._to_numpy(X)\n        if len(self.classes_) == 1:\n            # Handle cases where only one class was present in the training subset\n            probs = np.zeros((len(Xn), 2), dtype=np.float32)\n            if int(self.classes_[0]) < 2:\n                 probs[:, int(self.classes_[0])] = 1.0\n            return probs\n        \n        P = self.estimator.predict_proba(Xn)\n        \n        # Ensure output is always 2 columns for binary classification\n        if P.shape[1] == 1:\n            P1 = P[:, 0].astype(np.float32)\n            return np.column_stack([1.0 - P1, P1])\n        return P\n\n# --- Body Parts to Exclude ---\n# These are often redundant or noisy, derived from community insights.\nDROP_BODY_PARTS = [\n    'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright',\n    'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright',\n    'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T05:27:58.116386Z","iopub.execute_input":"2025-11-13T05:27:58.117208Z","iopub.status.idle":"2025-11-13T05:28:03.998549Z","shell.execute_reply.started":"2025-11-13T05:27:58.117168Z","shell.execute_reply":"2025-11-13T05:28:03.997724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# In[2]:\n# ===================================================================\n# Section 2: Data Loading and Preparation (with DEBUG mode fix)\n# ===================================================================\nprint(\"Loading data...\")\ntrain_df = pd.read_csv(os.path.join(CFG.BASE_PATH, 'train.csv'))\ntest_df = pd.read_csv(os.path.join(CFG.BASE_PATH, 'test.csv'))\n\nprint(\"Preparing training data...\")\n# Filter out the MABe22 datasets as they are mostly unannotated for behavior.\ntrain_df_filtered = train_df[~train_df['lab_id'].str.startswith('MABe22_')].copy()\n\n# In DEBUG mode, use a small, representative sample of the data.\nif CFG.DEBUG:\n    print(\"DEBUG mode enabled: Using a small sample of the training data.\")\n    \n    # FIX: Ensure the debug sample includes data for the public test set configuration.\n    test_config = test_df['body_parts_tracked'].iloc[0]\n    \n    matching_train_sample = train_df_filtered[train_df_filtered['body_parts_tracked'] == test_config]\n    if not matching_train_sample.empty:\n        matching_train_sample = matching_train_sample.sample(n=1, random_state=CFG.SEED)\n    \n    other_train_df = train_df_filtered[train_df_filtered['body_parts_tracked'] != test_config]\n    sample_size = max(1, int(len(train_df_filtered) * 0.05) - len(matching_train_sample))\n    \n    other_sample = other_train_df.sample(n=min(sample_size, len(other_train_df)), random_state=CFG.SEED)\n    \n    train_df_filtered = pd.concat([matching_train_sample, other_sample]).reset_index(drop=True)\n\n# Get a unique list of all body part configurations in the dataset.\nbody_parts_tracked_list = list(np.unique(train_df['body_parts_tracked']))\n\nprint(f\"Training on {len(train_df_filtered)} videos.\")\nprint(f\"Found {len(body_parts_tracked_list)} unique body part configurations.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T05:28:04.000002Z","iopub.execute_input":"2025-11-13T05:28:04.000833Z","iopub.status.idle":"2025-11-13T05:28:04.167933Z","shell.execute_reply.started":"2025-11-13T05:28:04.000804Z","shell.execute_reply":"2025-11-13T05:28:04.167108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# In[3]:\n# ===================================================================\n# Section 3: FPS-Aware Feature Engineering\n# ===================================================================\n\n# --- FPS Scaling Helper Functions ---\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    \"\"\"Scales a frame count from a 30 FPS reference to the actual video FPS.\"\"\"\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _fps_from_meta(meta_df):\n    \"\"\"Safely extracts FPS from a metadata dataframe.\"\"\"\n    return float(meta_df['frames_per_second'].iloc[0])\n\n# --- Feature Engineering Functions ---\n\ndef transform_single(single_mouse, body_parts_tracked, fps):\n    \"\"\"\n    Generates features for a single mouse. All temporal features are scaled by FPS.\n    \"\"\"\n    available_body_parts = single_mouse.columns.get_level_values(0)\n    X = pd.DataFrame(index=single_mouse.index)\n\n    # Base distance features (invariant to rotation/translation)\n    for p1, p2 in itertools.combinations(body_parts_tracked, 2):\n        if p1 in available_body_parts and p2 in available_body_parts:\n            X[f\"{p1}+{p2}\"] = np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n\n    # Kinematic features (speed, acceleration)\n    if 'body_center' in available_body_parts:\n        center_x = single_mouse['body_center']['x']\n        center_y = single_mouse['body_center']['y']\n        \n        speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n        acceleration = speed.diff() * float(fps)\n        \n        X['speed'] = speed\n        X['acceleration'] = acceleration\n        \n        for w in [15, 30, 60]: # ~0.5s, 1s, 2s windows at 30fps\n            ws = _scale(w, fps)\n            X[f'speed_mean_{w}'] = speed.rolling(ws, min_periods=1, center=True).mean()\n            X[f'speed_std_{w}'] = speed.rolling(ws, min_periods=1, center=True).std()\n            \n        for span in [30, 90, 180]:\n            s = _scale(span, fps)\n            X[f'speed_ema_{span}'] = speed.ewm(span=s, adjust=False).mean()\n\n    # Geometric features\n    if all(p in available_body_parts for p in ['nose', 'tail_base', 'ear_left', 'ear_right']):\n        X['elongation'] = X.get(f'nose+tail_base', np.nan) / (X.get(f'ear_left+ear_right', np.nan) + 1e-6)\n\n    if all(p in available_body_parts for p in ['nose', 'body_center', 'tail_base']):\n        v1 = single_mouse['nose'] - single_mouse['body_center']\n        v2 = single_mouse['tail_base'] - single_mouse['body_center']\n        dot = (v1['x'] * v2['x'] + v1['y'] * v2['y'])\n        norm = np.sqrt(v1['x']**2 + v1['y']**2) * np.sqrt(v2['x']**2 + v2['y']**2)\n        X['body_angle_cos'] = dot / (norm + 1e-6)\n\n    return X.astype(np.float32, copy=False)\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n    \"\"\"\n    Generates features for a pair of mice (A=agent, B=target).\n    \"\"\"\n    avail_A = mouse_pair['A'].columns.get_level_values(0)\n    avail_B = mouse_pair['B'].columns.get_level_values(0)\n    X = pd.DataFrame(index=mouse_pair.index)\n\n    # Inter-mouse distances\n    for p1 in body_parts_tracked:\n        for p2 in body_parts_tracked:\n            if p1 in avail_A and p2 in avail_B:\n                X[f\"12+{p1}+{p2}\"] = np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n\n    # Social interaction features\n    if 'body_center' in avail_A and 'body_center' in avail_B:\n        center_dist = np.sqrt(X.get(f'12+body_center+body_center', np.nan))\n        X['center_dist'] = center_dist\n        \n        for w in [15, 30, 60]:\n            ws = _scale(w, fps)\n            X[f'center_dist_mean_{w}'] = center_dist.rolling(ws, min_periods=1, center=True).mean()\n            X[f'center_dist_std_{w}'] = center_dist.rolling(ws, min_periods=1, center=True).std()\n            \n        X['approach_speed'] = -center_dist.diff() * float(fps)\n\n    # Relative orientation\n    if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n        dir_A = mouse_pair['A']['nose'] - mouse_pair['A']['tail_base']\n        dir_B = mouse_pair['B']['nose'] - mouse_pair['B']['tail_base']\n        dot = (dir_A['x'] * dir_B['x'] + dir_A['y'] * dir_B['y'])\n        norm = (np.sqrt(dir_A['x']**2 + dir_A['y']**2) * np.sqrt(dir_B['x']**2 + dir_B['y']**2))\n        X['relative_orientation_cos'] = dot / (norm + 1e-6)\n\n    return X.astype(np.float32, copy=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T05:28:04.168746Z","iopub.execute_input":"2025-11-13T05:28:04.169014Z","iopub.status.idle":"2025-11-13T05:28:04.184173Z","shell.execute_reply.started":"2025-11-13T05:28:04.168987Z","shell.execute_reply":"2025-11-13T05:28:04.183367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# In[4]:\n# ===================================================================\n# Section 4: Modeling and Prediction Pipeline (GPU-Enabled)\n# ===================================================================\n\n# --- Check for GPU availability ---\nGPU_AVAILABLE = False\ntry:\n    import torch\n    if torch.cuda.is_available():\n        GPU_AVAILABLE = True\n        print(\"GPU is available. Models will be trained on GPU.\")\n    else:\n        print(\"GPU not available. Models will be trained on CPU.\")\nexcept ImportError:\n    print(\"PyTorch not found. Models will be trained on CPU.\")\n\n\ndef generate_mouse_data(dataset, traintest, generate_single=True, generate_pair=True):\n    \"\"\"\n    Generator function to process video files one by one, yielding feature-ready dataframes.\n    This is the core of the memory-efficient pipeline.\n    \"\"\"\n    traintest_directory = os.path.join(CFG.BASE_PATH, f\"{traintest}_tracking\")\n    \n    for _, row in tqdm(dataset.iterrows(), total=len(dataset), desc=f\"Generating {traintest} data\"):\n        lab_id, video_id = row.lab_id, row.video_id\n        \n        if not isinstance(row.behaviors_labeled, str): continue\n\n        path = os.path.join(traintest_directory, lab_id, f\"{video_id}.parquet\")\n        if not os.path.exists(path): continue\n            \n        vid = pd.read_parquet(path)\n        if len(np.unique(vid.bodypart)) > 5:\n            vid = vid.query(\"~ bodypart.isin(@DROP_BODY_PARTS)\")\n        \n        pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n        del vid; gc.collect()\n        \n        pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T / row.pix_per_cm_approx\n\n        vid_behaviors = json.loads(row.behaviors_labeled)\n        vid_behaviors = [b.replace(\"'\", \"\").split(',') for b in sorted(list(set(vid_behaviors)))]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n\n        if traintest == 'train':\n            annot_path = path.replace('train_tracking', 'train_annotation')\n            if not os.path.exists(annot_path): continue\n            annot = pd.read_parquet(annot_path)\n\n        # Yield single-mouse data\n        if generate_single:\n            subset = vid_behaviors.query(\"target == 'self'\")\n            for mouse_id_str in np.unique(subset.agent):\n                try:\n                    mouse_id = int(mouse_id_str[-1])\n                    actions = np.unique(subset.query(\"agent == @mouse_id_str\").action)\n                    data = pvid.loc[:, mouse_id]\n                    meta = pd.DataFrame({\n                        'video_id': video_id, 'agent_id': mouse_id_str, 'target_id': 'self',\n                        'video_frame': data.index, 'frames_per_second': row.frames_per_second\n                    })\n                    if traintest == 'train':\n                        label = pd.DataFrame(0.0, columns=actions, index=data.index)\n                        annot_sub = annot.query(\"(agent_id == @mouse_id) & (target_id == @mouse_id)\")\n                        for _, r in annot_sub.iterrows():\n                            label.loc[r['start_frame']:r['stop_frame'], r.action] = 1.0\n                        yield 'single', data, meta, label\n                    else:\n                        yield 'single', data, meta, actions\n                except (KeyError, IndexError): continue\n\n        # Yield mouse-pair data\n        if generate_pair:\n            subset = vid_behaviors.query(\"target != 'self'\")\n            if not subset.empty:\n                for agent, target in itertools.permutations(np.unique(pvid.columns.get_level_values('mouse_id')), 2):\n                    agent_str, target_str = f\"mouse{agent}\", f\"mouse{target}\"\n                    actions = np.unique(subset.query(\"(agent == @agent_str) & (target == @target_str)\").action)\n                    if not actions.any(): continue\n                    try:\n                        data = pd.concat([pvid[agent], pvid[target]], axis=1, keys=['A', 'B'])\n                        meta = pd.DataFrame({\n                            'video_id': video_id, 'agent_id': agent_str, 'target_id': target_str,\n                            'video_frame': data.index, 'frames_per_second': row.frames_per_second\n                        })\n                        if traintest == 'train':\n                            label = pd.DataFrame(0.0, columns=actions, index=data.index)\n                            annot_sub = annot.query(\"(agent_id == @agent) & (target_id == @target)\")\n                            for _, r in annot_sub.iterrows():\n                                label.loc[r['start_frame']:r['stop_frame'], r.action] = 1.0\n                            yield 'pair', data, meta, label\n                        else:\n                            yield 'pair', data, meta, actions\n                    except (KeyError, IndexError): continue\n\ndef predict_multiclass_adaptive(pred, meta, thresholds):\n    \"\"\"FIXED: Correctly converts frame-wise probabilities into start/stop segments.\"\"\"\n    pred_smoothed = pred.rolling(window=5, min_periods=1, center=True).mean()\n    \n    ama = np.argmax(pred_smoothed.values, axis=1)\n    max_probs = pred_smoothed.max(axis=1).values\n    \n    threshold_array = np.array([thresholds.get(col, CFG.PREDICTION_THRESHOLD) for col in pred.columns])\n    action_thresholds = threshold_array[ama]\n\n    ama = np.where(max_probs >= action_thresholds, ama, -1)\n    ama = pd.Series(ama, index=meta.video_frame)\n\n    changes = (ama != ama.shift(1))\n    change_indices = meta.loc[changes, 'video_frame'].tolist()\n    actions_at_changes = ama[changes].tolist()\n    \n    if not change_indices or change_indices[-1] != meta['video_frame'].max():\n         change_indices.append(meta['video_frame'].max() + 1)\n         actions_at_changes.append(-1)\n\n    starts, stops, action_indices = [], [], []\n    for i in range(len(actions_at_changes) - 1):\n        if actions_at_changes[i] >= 0:\n            starts.append(change_indices[i])\n            stops.append(change_indices[i+1])\n            action_indices.append(actions_at_changes[i])\n            \n    if not starts:\n        return pd.DataFrame()\n\n    submission_part = pd.DataFrame({'action': pred.columns[action_indices], 'start_frame': starts, 'stop_frame': stops})\n    \n    meta_deduped = meta.drop_duplicates(subset=['video_id', 'agent_id', 'target_id'])\n    submission_part['video_id'] = meta_deduped['video_id'].iloc[0]\n    submission_part['agent_id'] = meta_deduped['agent_id'].iloc[0]\n    submission_part['target_id'] = meta_deduped['target_id'].iloc[0]\n\n    submission_part = submission_part[submission_part['stop_frame'] - submission_part['start_frame'] >= 3]\n    return submission_part[['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']]\n\ndef submit_ensemble(body_parts_tracked_str, switch_tr, X_tr, label, meta, n_samples, train_feature_cols):\n    global submission_list\n    \n    # --- GPU-Aware Model Ensemble Definition ---\n    models = []\n    lgbm_params = {'random_state': CFG.SEED, 'n_jobs': -1, 'force_col_wise': True}\n    xgb_params = {'random_state': CFG.SEED, 'n_jobs': -1}\n\n    if GPU_AVAILABLE:\n        print(\"  Configuring models for GPU...\")\n        lgbm_params['device_type'] = 'gpu'\n        xgb_params['tree_method'] = 'gpu_hist'\n    else:\n        print(\"  Configuring models for CPU...\")\n        xgb_params['tree_method'] = 'hist'\n\n    models.append(make_pipeline(SimpleImputer(), StratifiedSubsetClassifier(\n        lgb.LGBMClassifier(n_estimators=200, learning_rate=0.07, **lgbm_params), n_samples)))\n    \n    models.append(make_pipeline(SimpleImputer(), StratifiedSubsetClassifier(\n        XGBClassifier(n_estimators=180, learning_rate=0.08, max_depth=6, **xgb_params), n_samples)))\n\n    # --- Train one ensemble per action ---\n    model_list = []\n    for action in label.columns:\n        action_mask = ~label[action].isna()\n        y_action = label.loc[action_mask, action].values.astype(int)\n        \n        if np.sum(y_action) >= 5 and len(y_action) - np.sum(y_action) >= 5:\n            trained_models = [clone(m).fit(X_tr[action_mask], y_action) for m in models]\n            model_list.append((action, trained_models))\n\n    del X_tr; gc.collect()\n\n    # --- Inference on Test Set ---\n    test_subset = test_df[test_df.body_parts_tracked == body_parts_tracked_str]\n    if test_subset.empty: return\n    \n    generator = generate_mouse_data(test_subset, 'test', \n                                    generate_single=(switch_tr == 'single'), \n                                    generate_pair=(switch_tr == 'pair'))\n\n    for switch_te, data_te, meta_te, actions_te in generator:\n        try:\n            fps = _fps_from_meta(meta_te)\n            body_parts_tracked = json.loads(body_parts_tracked_str)\n            X_te = transform_single(data_te, body_parts_tracked, fps) if switch_te == 'single' else transform_pair(data_te, body_parts_tracked, fps)\n            \n            # FIX: Align test features with training features\n            X_te = X_te.reindex(columns=train_feature_cols, fill_value=np.nan)\n            \n            del data_te; gc.collect()\n\n            pred = pd.DataFrame(index=meta_te.video_frame)\n            for action, trained_models in model_list:\n                if action in actions_te:\n                    probs = [m.predict_proba(X_te)[:, 1] for m in trained_models]\n                    pred[action] = np.mean(probs, axis=0)\n            del X_te; gc.collect()\n            \n            if not pred.empty:\n                submission_part = predict_multiclass_adaptive(pred, meta_te, CFG.ACTION_THRESHOLDS)\n                submission_list.append(submission_part)\n        except Exception as e:\n            print(f\"  ERROR during inference: {e}\")\n            continue","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T05:28:04.185402Z","iopub.execute_input":"2025-11-13T05:28:04.185726Z","iopub.status.idle":"2025-11-13T05:28:07.908916Z","shell.execute_reply.started":"2025-11-13T05:28:04.185699Z","shell.execute_reply":"2025-11-13T05:28:07.908287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# In[5]:\n# ===================================================================\n# Section 5: Main Execution Loop and Submission Generation\n# ===================================================================\n\ndef robustify(submission, dataset):\n    \"\"\"\n    Cleans the submission file by removing overlaps and ensuring every video\n    has at least one prediction to prevent scoring errors.\n    \"\"\"\n    submission = submission[submission.start_frame < submission.stop_frame].copy()\n\n    group_list = []\n    for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n        group = group.sort_values('start_frame').reset_index(drop=True)\n        if not group.empty:\n            last_stop_frame = 0\n            for i, row in group.iterrows():\n                if row['start_frame'] < last_stop_frame:\n                    group.at[i, 'start_frame'] = last_stop_frame\n                last_stop_frame = row['stop_frame']\n        group_list.append(group)\n    \n    submission = pd.concat(group_list).reset_index(drop=True)\n    submission = submission[submission.start_frame < submission.stop_frame]\n\n    predicted_videos = set(submission['video_id'].unique())\n    all_test_videos = set(dataset['video_id'].unique())\n    missing_videos = all_test_videos - predicted_videos\n    \n    if missing_videos:\n        print(f\"Found {len(missing_videos)} videos with no predictions. Adding placeholders.\")\n        placeholders = []\n        for video_id in missing_videos:\n            behaviors_str = dataset[dataset.video_id == video_id]['behaviors_labeled'].iloc[0]\n            if isinstance(behaviors_str, str) and behaviors_str != '[]':\n                agent, target, action = json.loads(behaviors_str)[0].replace(\"'\", \"\").split(',')\n                placeholders.append({'video_id': video_id, 'agent_id': agent, 'target_id': target,\n                                     'action': action, 'start_frame': 0, 'stop_frame': 100})\n        if placeholders:\n            submission = pd.concat([submission, pd.DataFrame(placeholders)], ignore_index=True)\n            \n    return submission.reset_index(drop=True)\n\n# --- Main Loop ---\nsubmission_list = []\nloop_iterator = range(len(body_parts_tracked_list))\n\nfor section in loop_iterator:\n    if section == 0: continue\n    \n    body_parts_tracked_str = body_parts_tracked_list[section]\n    body_parts_tracked = json.loads(body_parts_tracked_str)\n    print(f\"\\n--- Processing Section {section}/{len(body_parts_tracked_list)-1}: {len(body_parts_tracked)} body parts ---\")\n\n    train_subset = train_df_filtered[train_df_filtered.body_parts_tracked == body_parts_tracked_str]\n    if train_subset.empty:\n        print(\"No training data for this configuration.\")\n        continue\n\n    single_mouse_list, single_label_list, single_meta_list = [], [], []\n    pair_list, pair_label_list, pair_meta_list = [], [], []\n\n    data_generator = generate_mouse_data(train_subset, 'train')\n    for switch, data, meta, label in data_generator:\n        (single_mouse_list if switch == 'single' else pair_list).append(data)\n        (single_meta_list if switch == 'single' else pair_meta_list).append(meta)\n        (single_label_list if switch == 'single' else pair_label_list).append(label)\n    \n    if single_mouse_list:\n        print(f\"Processing {len(single_mouse_list)} single-mouse batches...\")\n        all_data = pd.concat(single_mouse_list); all_meta = pd.concat(single_meta_list); all_labels = pd.concat(single_label_list)\n        del single_mouse_list, single_meta_list, single_label_list; gc.collect()\n        \n        fps = _fps_from_meta(all_meta)\n        X_tr = transform_single(all_data, body_parts_tracked, fps)\n        train_feature_cols = X_tr.columns.tolist() # Capture feature names\n        \n        print(f\"  Training on single-mouse data with shape {X_tr.shape}\")\n        submit_ensemble(body_parts_tracked_str, 'single', X_tr, all_labels, all_meta, CFG.N_SAMPLES_SINGLE, train_feature_cols)\n        del X_tr, all_data, all_meta, all_labels; gc.collect()\n\n    if pair_list:\n        print(f\"Processing {len(pair_list)} mouse-pair batches...\")\n        all_data = pd.concat(pair_list); all_meta = pd.concat(pair_meta_list); all_labels = pd.concat(pair_label_list)\n        del pair_list, pair_meta_list, pair_label_list; gc.collect()\n        \n        fps = _fps_from_meta(all_meta)\n        X_tr = transform_pair(all_data, body_parts_tracked, fps)\n        train_feature_cols = X_tr.columns.tolist() # Capture feature names\n        \n        print(f\"  Training on mouse-pair data with shape {X_tr.shape}\")\n        submit_ensemble(body_parts_tracked_str, 'pair', X_tr, all_labels, all_meta, CFG.N_SAMPLES_PAIR, train_feature_cols)\n        del X_tr, all_data, all_meta, all_labels; gc.collect()\n\n# --- Finalize and Save Submission ---\nprint(\"\\n--- Finalizing Submission ---\")\nif submission_list:\n    submission_df = pd.concat(submission_list, ignore_index=True)\nelse:\n    submission_df = pd.DataFrame([{'video_id': test_df['video_id'].iloc[0], 'agent_id': 'mouse1', \n                                   'target_id': 'self', 'action': 'rear', 'start_frame': 0, 'stop_frame': 100}])\n\nprint(f\"Generated {len(submission_df)} raw predictions.\")\n\nsubmission_final = robustify(submission_df, test_df)\nsubmission_final.index.name = 'row_id'\nsubmission_final.to_csv('submission.csv')\n\nprint(f\"Final submission saved to 'submission.csv' with {len(submission_final)} rows.\")\nprint(\"Done!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T05:28:07.910149Z","iopub.execute_input":"2025-11-13T05:28:07.910634Z","iopub.status.idle":"2025-11-13T05:31:22.919624Z","shell.execute_reply.started":"2025-11-13T05:28:07.910612Z","shell.execute_reply":"2025-11-13T05:31:22.918881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}