{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13719397,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:40:12.381285Z","iopub.execute_input":"2025-09-26T17:40:12.382143Z","iopub.status.idle":"2025-09-26T17:40:15.282317Z","shell.execute_reply.started":"2025-09-26T17:40:12.382105Z","shell.execute_reply":"2025-09-26T17:40:15.28144Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport json\nimport itertools\nimport warnings\nfrom sklearn.model_selection import cross_val_predict, GroupKFold\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.metrics import f1_score\nfrom sklearn.base import clone\nfrom sklearn.ensemble import VotingClassifier, RandomForestClassifier\nfrom lightgbm import LGBMClassifier\n\n# ============= CONFIGURATION =============\nvalidate_or_submit = 'submit' \nverbose = True\n\n# ============= LOAD DATA =============\nprint(\"Loading training and test metadata...\")\ntrain = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/train.csv')\ntrain['n_mice'] = 4 - train[['mouse1_strain', 'mouse2_strain', 'mouse3_strain', 'mouse4_strain']].isna().sum(axis=1)\ntrain_without_mabe22 = train.query(\"~ lab_id.str.startswith('MABe22_')\")\n\ntest = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/test.csv')\nbody_parts_tracked_list = list(np.unique(train.body_parts_tracked))\n\nprint(f\"Found {len(body_parts_tracked_list)} unique body part configurations\")\n\n# ============= ENHANCED UTILITY CLASSES =============\nclass TrainOnSubsetClassifier:\n    \"\"\"Wrapper to train classifier on subset for memory efficiency\"\"\"\n    def __init__(self, clf, max_samples):\n        self.clf = clf\n        self.max_samples = max_samples\n    \n    def fit(self, X, y):\n        if len(X) > self.max_samples:\n            idx = np.random.choice(len(X), self.max_samples, replace=False)\n            X_subset = X[idx] if hasattr(X, 'iloc') else X[idx]\n            y_subset = y[idx] if hasattr(y, 'iloc') else y[idx]\n        else:\n            X_subset, y_subset = X, y\n        \n        self.clf.fit(X_subset, y_subset)\n        return self\n    \n    def predict_proba(self, X):\n        return self.clf.predict_proba(X)\n    \n    @property\n    def classes_(self):\n        return self.clf.classes_\n\n# ============= ENHANCED FEATURE ENGINEERING =============\ndef transform_single_enhanced(single_mouse, body_parts_tracked):\n    \"\"\"Enhanced transform for single mouse with temporal features\"\"\"\n    \n    # Original distance features\n    X = pd.DataFrame({\n        f\"{part1}+{part2}\": np.square(single_mouse[part1] - single_mouse[part2]).sum(axis=1, skipna=False)\n        for part1, part2 in itertools.combinations(body_parts_tracked, 2)\n    })\n    \n    # Enhanced temporal features with additional windows\n    if 'ear_left' in single_mouse.columns and 'ear_right' in single_mouse.columns and 'tail_base' in single_mouse.columns:\n        # Speed features at multiple scales\n        for window in [5, 10, 15, 20]:\n            shifted = single_mouse[['ear_left', 'ear_right', 'tail_base']].shift(window)\n            speed_features = pd.DataFrame({\n                f'speed_left_{window}': np.square(single_mouse['ear_left'] - shifted['ear_left']).sum(axis=1, skipna=False),\n                f'speed_right_{window}': np.square(single_mouse['ear_right'] - shifted['ear_right']).sum(axis=1, skipna=False),\n                f'speed_tail_{window}': np.square(single_mouse['tail_base'] - shifted['tail_base']).sum(axis=1, skipna=False),\n            })\n            X = pd.concat([X, speed_features], axis=1)\n        \n        # Acceleration features\n        shifted_10 = single_mouse[['ear_left', 'ear_right']].shift(10)\n        shifted_20 = single_mouse[['ear_left', 'ear_right']].shift(20)\n        if not shifted_20.isna().all().all():\n            speed_10 = np.square(single_mouse['ear_left'] - shifted_10['ear_left']).sum(axis=1, skipna=False)\n            speed_20 = np.square(shifted_10['ear_left'] - shifted_20['ear_left']).sum(axis=1, skipna=False)\n            accel_features = pd.DataFrame({\n                'accel_left': speed_10 - speed_20,\n                'accel_right': np.square(single_mouse['ear_right'] - shifted_10['ear_right']).sum(axis=1, skipna=False) - \n                              np.square(shifted_10['ear_right'] - shifted_20['ear_right']).sum(axis=1, skipna=False),\n            })\n            X = pd.concat([X, accel_features], axis=1)\n    \n    # Add body orientation angle if nose and tail_base available\n    if 'nose' in single_mouse.columns and 'tail_base' in single_mouse.columns:\n        dx = single_mouse['nose'].iloc[:, 0] - single_mouse['tail_base'].iloc[:, 0]\n        dy = single_mouse['nose'].iloc[:, 1] - single_mouse['tail_base'].iloc[:, 1]\n        body_angle = np.arctan2(dy, dx)\n        \n        X['body_angle'] = body_angle\n        X['body_angle_change'] = body_angle.diff(5)  # Angular velocity\n        \n        # Body length (useful for rearing, stretching behaviors)\n        body_length = np.sqrt(np.square(single_mouse['nose'] - single_mouse['tail_base']).sum(axis=1, skipna=False))\n        X['body_length'] = body_length\n        X['body_length_change'] = body_length.diff(10)\n    \n    # Add centroid-based features for overall movement\n    if 'nose' in single_mouse.columns:\n        # Simple centroid using available parts\n        x_coords = []\n        y_coords = []\n        for part in ['nose', 'tail_base'] if 'tail_base' in single_mouse.columns else ['nose']:\n            if part in single_mouse.columns:\n                x_coords.append(single_mouse[part].iloc[:, 0])\n                y_coords.append(single_mouse[part].iloc[:, 1])\n        \n        if x_coords:\n            centroid_x = pd.concat(x_coords, axis=1).mean(axis=1)\n            centroid_y = pd.concat(y_coords, axis=1).mean(axis=1)\n            \n            # Centroid speed\n            for window in [10, 20]:\n                centroid_shift_x = centroid_x.shift(window)\n                centroid_shift_y = centroid_y.shift(window)\n                centroid_speed = np.sqrt(np.square(centroid_x - centroid_shift_x) + \n                                        np.square(centroid_y - centroid_shift_y))\n                X[f'centroid_speed_{window}'] = centroid_speed\n    \n    return X\n\ndef transform_pair_enhanced(mouse_pair, body_parts_tracked):\n    \"\"\"Enhanced transform for mouse pairs with social features\"\"\"\n    \n    # Filter body parts for memory efficiency\n    drop_body_parts = ['ear_left', 'ear_right',\n                      'headpiece_bottombackleft', 'headpiece_bottombackright', \n                      'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n                      'headpiece_topbackleft', 'headpiece_topbackright', \n                      'headpiece_topfrontleft', 'headpiece_topfrontright', \n                      'tail_midpoint']\n    \n    if len(body_parts_tracked) > 5:\n        body_parts_tracked = [b for b in body_parts_tracked if b not in drop_body_parts]\n    \n    # Original inter-mouse distance features\n    X = pd.DataFrame({\n        f\"12+{part1}+{part2}\": np.square(mouse_pair['A'][part1] - mouse_pair['B'][part2]).sum(axis=1, skipna=False)\n        for part1, part2 in itertools.product(body_parts_tracked, repeat=2)\n    })\n\n    # Enhanced social interaction features\n    if 'nose' in body_parts_tracked and 'tail_base' in body_parts_tracked:\n        # Face-to-face distance (important for social behaviors)\n        X['face_distance'] = np.square(mouse_pair['A']['nose'] - mouse_pair['B']['nose']).sum(axis=1, skipna=False)\n        # Following behavior indicators\n        X['following_AB'] = np.square(mouse_pair['A']['nose'] - mouse_pair['B']['tail_base']).sum(axis=1, skipna=False)\n        X['following_BA'] = np.square(mouse_pair['B']['nose'] - mouse_pair['A']['tail_base']).sum(axis=1, skipna=False)\n        \n        # Add approach/retreat dynamics\n        face_dist = X['face_distance']\n        X['face_distance_change_5'] = face_dist.diff(5)\n        X['face_distance_change_10'] = face_dist.diff(10)\n        \n        # Relative orientation (are mice facing each other?)\n        dx_A = mouse_pair['A']['nose'].iloc[:, 0] - mouse_pair['A']['tail_base'].iloc[:, 0]\n        dy_A = mouse_pair['A']['nose'].iloc[:, 1] - mouse_pair['A']['tail_base'].iloc[:, 1]\n        angle_A = np.arctan2(dy_A, dx_A)\n        \n        dx_B = mouse_pair['B']['nose'].iloc[:, 0] - mouse_pair['B']['tail_base'].iloc[:, 0]\n        dy_B = mouse_pair['B']['nose'].iloc[:, 1] - mouse_pair['B']['tail_base'].iloc[:, 1]\n        angle_B = np.arctan2(dy_B, dx_B)\n        \n        X['relative_angle'] = np.cos(angle_A - angle_B)  # 1 = same direction, -1 = opposite\n\n    # Enhanced speed features with multiple windows\n    if ('A', 'ear_left') in mouse_pair.columns and ('B', 'ear_left') in mouse_pair.columns:\n        for window in [5, 10, 15]:\n            shifted_A = mouse_pair['A']['ear_left'].shift(window)\n            shifted_B = mouse_pair['B']['ear_left'].shift(window)\n            X[f'speed_left_A_{window}'] = np.square(mouse_pair['A']['ear_left'] - shifted_A).sum(axis=1, skipna=False)\n            X[f'speed_left_AB_{window}'] = np.square(mouse_pair['A']['ear_left'] - shifted_B).sum(axis=1, skipna=False)\n            X[f'speed_left_B_{window}'] = np.square(mouse_pair['B']['ear_left'] - shifted_B).sum(axis=1, skipna=False)\n    \n    # Add centroid distance and changes\n    if 'nose' in body_parts_tracked and 'tail_base' in body_parts_tracked:\n        # Compute simple centroids\n        centroid_A_x = (mouse_pair['A']['nose'].iloc[:, 0] + mouse_pair['A']['tail_base'].iloc[:, 0]) / 2\n        centroid_A_y = (mouse_pair['A']['nose'].iloc[:, 1] + mouse_pair['A']['tail_base'].iloc[:, 1]) / 2\n        centroid_B_x = (mouse_pair['B']['nose'].iloc[:, 0] + mouse_pair['B']['tail_base'].iloc[:, 0]) / 2\n        centroid_B_y = (mouse_pair['B']['nose'].iloc[:, 1] + mouse_pair['B']['tail_base'].iloc[:, 1]) / 2\n        \n        centroid_dist = np.sqrt(np.square(centroid_A_x - centroid_B_x) + np.square(centroid_A_y - centroid_B_y))\n        X['centroid_distance'] = centroid_dist\n        X['centroid_dist_change_10'] = centroid_dist.diff(10)\n        X['centroid_dist_change_20'] = centroid_dist.diff(20)\n    \n    return X\n\n# ============= DYNAMIC THRESHOLD OPTIMIZATION =============\ndef find_optimal_thresholds(oof_predictions, labels, default_threshold=0.27):\n    \"\"\"Find optimal threshold for each action\"\"\"\n    optimal_thresholds = {}\n    \n    for action in oof_predictions.columns:\n        if action in labels.columns:\n            mask = ~labels[action].isna()\n            if mask.sum() > 100:  # Sufficient data for optimization\n                y_true = labels[action][mask].values.astype(int)\n                y_pred_proba = oof_predictions[action][mask].values\n                \n                best_f1 = 0\n                best_thresh = default_threshold\n                \n                # Quick grid search\n                for thresh in [0.15, 0.2, 0.25, 0.27, 0.3, 0.35, 0.4, 0.45, 0.5]:\n                    f1 = f1_score(y_true, y_pred_proba >= thresh, zero_division=0)\n                    if f1 > best_f1:\n                        best_f1 = f1\n                        best_thresh = thresh\n                        \n                optimal_thresholds[action] = best_thresh\n            else:\n                optimal_thresholds[action] = default_threshold\n    \n    return optimal_thresholds\n\n# ============= ENHANCED MODEL CREATION =============\ndef create_simple_ensemble():\n    \"\"\"Create simple but effective ensemble\"\"\"\n    lgb = LGBMClassifier(\n        n_estimators=500,\n        max_depth=6, \n        learning_rate=0.05,\n        random_state=42,\n        verbosity=-1,\n        force_row_wise=True\n    )\n    \n    rf = RandomForestClassifier(\n        n_estimators=300,\n        max_depth=8,\n        random_state=42,\n        n_jobs=-1\n    )\n    \n    ensemble = VotingClassifier([('lgb', lgb), ('rf', rf)], voting='soft')\n    \n    return make_pipeline(\n        SimpleImputer(),\n        StandardScaler(), \n        TrainOnSubsetClassifier(ensemble, 25000)\n    )\n\n# ============= ENHANCED MULTICLASS PREDICTION =============\ndef predict_multiclass_optimized(pred, meta, thresholds=None):\n    \"\"\"Enhanced multiclass prediction with optimized thresholds\"\"\"\n    if thresholds is None:\n        thresholds = {col: 0.27 for col in pred.columns}\n    \n    # Apply action-specific thresholds\n    ama = np.argmax(pred.values, axis=1)\n    max_proba = pred.max(axis=1).values\n    \n    # Use action-specific thresholds\n    threshold_array = np.array([thresholds.get(col, 0.27) for col in pred.columns])\n    action_thresholds = threshold_array[ama]\n    \n    ama = np.where(max_proba >= action_thresholds, ama, -1)\n    ama = pd.Series(ama, index=meta.video_frame)\n    \n    # Keep only start and stop frames\n    changes_mask = (ama != ama.shift(1)).values\n    ama_changes = ama[changes_mask]\n    meta_changes = meta[changes_mask]\n    \n    # mask selects the start frames\n    mask = ama_changes.values >= 0  # start of action\n    mask[-1] = False\n    \n    submission_part = pd.DataFrame({\n        'video_id': meta_changes['video_id'][mask].values,\n        'agent_id': meta_changes['agent_id'][mask].values,\n        'target_id': meta_changes['target_id'][mask].values,\n        'action': pred.columns[ama_changes[mask].values],\n        'start_frame': ama_changes.index[mask],\n        'stop_frame': ama_changes.index[1:][mask[:-1]]\n    })\n    \n    # Fix stop_frame for video boundaries\n    stop_video_id = meta_changes['video_id'][1:][mask[:-1]].values\n    stop_agent_id = meta_changes['agent_id'][1:][mask[:-1]].values\n    stop_target_id = meta_changes['target_id'][1:][mask[:-1]].values\n    \n    for i in range(len(submission_part)):\n        video_id = submission_part.video_id.iloc[i]\n        agent_id = submission_part.agent_id.iloc[i]\n        target_id = submission_part.target_id.iloc[i]\n        if stop_video_id[i] != video_id or stop_agent_id[i] != agent_id or stop_target_id[i] != target_id:\n            new_stop_frame = meta.query(\"(video_id == @video_id)\").video_frame.max() + 1\n            submission_part.iat[i, submission_part.columns.get_loc('stop_frame')] = new_stop_frame\n    \n    assert (submission_part.stop_frame > submission_part.start_frame).all(), 'stop <= start'\n    if verbose: \n        print('  actions found:', len(submission_part))\n        \n    return submission_part\n\n# ============= DATA GENERATION =============\ndef generate_mouse_data(dataset, traintest, traintest_directory=None, generate_single=True, generate_pair=True):\n    \"\"\"Generate batches of data in coordinate representation\"\"\"\n    \n    assert traintest in ['train', 'test']\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n        \n    for _, row in dataset.iterrows():\n        # Load the video and pivot it so that one frame = one row\n        lab_id = row.lab_id\n        if lab_id.startswith('MABe22'): \n            continue\n        video_id = row.video_id\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        \n        vid = pd.read_parquet(path)\n        pvid = vid.pivot(columns=['mouse_id', 'bodypart'], index='video_frame', values=['x', 'y'])\n        \n        if pvid.isna().any().any():\n            if verbose and traintest == 'test': \n                print('video with missing values', video_id, traintest, len(vid), 'frames')\n        else:\n            if verbose and traintest == 'test': \n                print('video with all values', video_id, traintest, len(vid), 'frames')\n        \n        del vid\n        pvid = pvid.reorder_levels([1, 2, 0], axis=1).T.sort_index().T  # mouse_id, body_part, xy\n        pvid /= row.pix_per_cm_approx  # convert to cm\n\n        # Determine the behaviors of this video\n        vid_behaviors = json.loads(row.behaviors_labeled)\n        vid_behaviors = sorted(list({b.replace(\"'\", \"\") for b in vid_behaviors}))\n        vid_behaviors = [b.split(',') for b in vid_behaviors]\n        vid_behaviors = pd.DataFrame(vid_behaviors, columns=['agent', 'target', 'action'])\n        \n        # Load the annotations for training\n        if traintest == 'train':\n            try:\n                annot = pd.read_parquet(path.replace('train_tracking', 'train_annotation'))\n            except FileNotFoundError:\n                # MABe22 and one more training file lack annotations\n                continue\n\n        # Create single_mouse dataframes\n        if generate_single:\n            vid_behaviors_subset = vid_behaviors.query(\"target == 'self'\")\n            for mouse_id_str in np.unique(vid_behaviors_subset.agent):\n                try:\n                    mouse_id = int(mouse_id_str[-1])\n                    vid_agent_actions = np.unique(vid_behaviors_subset.query(\"agent == @mouse_id_str\").action)\n                    single_mouse = pvid.loc[:, mouse_id]\n                    \n                    single_mouse_meta = pd.DataFrame({\n                        'video_id': video_id,\n                        'agent_id': mouse_id_str,\n                        'target_id': 'self',\n                        'video_frame': single_mouse.index\n                    })\n                    \n                    if traintest == 'train':\n                        single_mouse_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=single_mouse.index)\n                        annot_subset = annot.query(\"(agent_id == @mouse_id) & (target_id == @mouse_id)\")\n                        for i in range(len(annot_subset)):\n                            annot_row = annot_subset.iloc[i]\n                            single_mouse_label.loc[annot_row['start_frame']:annot_row['stop_frame'], annot_row.action] = 1.0\n                        yield 'single', single_mouse, single_mouse_meta, single_mouse_label\n                    else:\n                        if verbose: \n                            print('- test single', video_id, mouse_id)\n                        yield 'single', single_mouse, single_mouse_meta, vid_agent_actions\n                except KeyError:\n                    pass  # Skip if no data for selected agent mouse\n\n        # Create mouse_pair dataframes\n        if generate_pair:\n            vid_behaviors_subset = vid_behaviors.query(\"target != 'self'\")\n            if len(vid_behaviors_subset) > 0:\n                for agent, target in itertools.permutations(np.unique(pvid.columns.get_level_values('mouse_id')), 2):\n                    agent_str = f\"mouse{agent}\"\n                    target_str = f\"mouse{target}\"\n                    vid_agent_actions = np.unique(vid_behaviors_subset.query(\"(agent == @agent_str) & (target == @target_str)\").action)\n                    \n                    mouse_pair = pd.concat([pvid[agent], pvid[target]], axis=1, keys=['A', 'B'])\n                    mouse_pair_meta = pd.DataFrame({\n                        'video_id': video_id,\n                        'agent_id': agent_str,\n                        'target_id': target_str,\n                        'video_frame': mouse_pair.index\n                    })\n                    \n                    if traintest == 'train':\n                        mouse_pair_label = pd.DataFrame(0.0, columns=vid_agent_actions, index=mouse_pair.index)\n                        annot_subset = annot.query(\"(agent_id == @agent) & (target_id == @target)\")\n                        for i in range(len(annot_subset)):\n                            annot_row = annot_subset.iloc[i]\n                            mouse_pair_label.loc[annot_row['start_frame']:annot_row['stop_frame'], annot_row.action] = 1.0\n                        yield 'pair', mouse_pair, mouse_pair_meta, mouse_pair_label\n                    else:\n                        if verbose: \n                            print('- test pair', video_id, agent, target)\n                        yield 'pair', mouse_pair, mouse_pair_meta, vid_agent_actions\n\n# ============= ENHANCED CROSS-VALIDATION =============\ndef cross_validate_classifier_enhanced(binary_classifier, X, label, meta):\n    \"\"\"Enhanced cross-validation with optimized thresholds\"\"\"\n    \n    oof = pd.DataFrame(index=meta.video_frame)\n    \n    for action in label.columns:\n        # Filter for samples with defined target\n        action_mask = ~label[action].isna().values\n        X_action = X[action_mask]\n        y_action = label[action][action_mask].values.astype(int)\n        p = y_action.mean()\n        baseline_score = p / (1 + p)\n        groups_action = meta.video_id[action_mask]\n        \n        if len(np.unique(groups_action)) < 3:  # Need at least 3 groups for 3-fold CV\n            continue\n            \n        if ~(y_action == 0).all():\n            with warnings.catch_warnings():\n                warnings.filterwarnings('ignore', category=RuntimeWarning)\n                oof_action = cross_val_predict(\n                    binary_classifier, X_action, y_action, \n                    groups=groups_action, cv=GroupKFold(n_splits=3),  # Reduced to 3-fold\n                    method='predict_proba'\n                )\n            oof_action = oof_action[:, 1]\n        else:\n            oof_action = np.zeros(len(y_action))\n            \n        # Store OOF predictions\n        oof_column = np.zeros(len(label))\n        oof_column[action_mask] = oof_action\n        oof[action] = oof_column\n\n    # Find optimal thresholds\n    optimal_thresholds = find_optimal_thresholds(oof, label)\n    \n    if verbose:\n        print(\"Optimal thresholds:\", {k: f\"{v:.3f}\" for k, v in optimal_thresholds.items()})\n    \n    # Make multiclass prediction with optimized thresholds\n    submission_part = predict_multiclass_optimized(oof, meta, optimal_thresholds)\n    submission_list.append(submission_part)\n\n# ============= SUBMISSION GENERATION =============\ndef submit_enhanced(body_parts_tracked_str, switch_tr, binary_classifier, X_tr, label, meta):\n    \"\"\"Enhanced submission with optimized thresholds\"\"\"\n    \n    # Fit binary classifier for every action\n    model_list = []\n    for action in label.columns:\n        action_mask = ~label[action].isna().values\n        y_action = label[action][action_mask].values.astype(int)\n\n        if ~(y_action == 0).all():\n            model = clone(binary_classifier)\n            model.fit(X_tr[action_mask], y_action)\n            assert len(model.classes_) == 2\n            model_list.append((action, model))\n\n    # Compute test predictions in batches\n    body_parts_tracked = json.loads(body_parts_tracked_str)\n    \n    if validate_or_submit == 'submit':\n        test_subset = test[test.body_parts_tracked == body_parts_tracked_str]\n        generator = generate_mouse_data(test_subset, 'test',\n                                      generate_single=(switch_tr == 'single'), \n                                      generate_pair=(switch_tr == 'pair'))\n    else:\n        test_subset = stresstest.query(\"body_parts_tracked == @body_parts_tracked_str\")\n        generator = generate_mouse_data(test_subset, 'test',\n                                      traintest_directory='stresstest_tracking',\n                                      generate_single=(switch_tr == 'single'),\n                                      generate_pair=(switch_tr == 'pair'))\n    \n    if verbose: \n        print(f\"n_videos: {len(test_subset)}\")\n    \n    for switch_te, data_te, meta_te, actions_te in generator:\n        assert switch_te == switch_tr\n        try:\n            # Transform from coordinate to distance representation\n            if switch_te == 'single':\n                X_te = transform_single_enhanced(data_te, body_parts_tracked)\n            else:\n                X_te = transform_pair_enhanced(data_te, body_parts_tracked)\n                \n            if verbose and len(X_te) == 0: \n                print(\"ERROR: X_te is empty\")\n            del data_te\n\n            # Compute binary predictions\n            pred = pd.DataFrame(index=meta_te.video_frame)\n            for action, model in model_list:\n                if action in actions_te:\n                    pred[action] = model.predict_proba(X_te)[:, 1]\n            del X_te\n\n            # Compute multiclass predictions with default thresholds\n            if pred.shape[1] != 0:\n                submission_part = predict_multiclass_optimized(pred, meta_te)\n                submission_list.append(submission_part)\n            else:\n                if verbose: \n                    print(f\"  ERROR: no useful training data\")\n                    \n        except KeyError:\n            if verbose: \n                print(f'  ERROR: KeyError because of missing bodypart ({switch_tr})')\n            if 'data_te' in locals():\n                del data_te\n\n# ============= ROBUSTIFICATION =============\ndef robustify(submission, dataset, traintest, traintest_directory=None):\n    \"\"\"Ensure submission conforms to competition rules\"\"\"\n    \n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n    # Rule 1: Ensure that start_frame < stop_frame\n    old_submission = submission.copy()\n    submission = submission[submission.start_frame < submission.stop_frame]\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped frames with start >= stop\")\n    \n    # Rule 2: Avoid multiple predictions for same frame from one agent/target pair\n    old_submission = submission.copy()\n    group_list = []\n    for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n        group = group.sort_values('start_frame')\n        mask = np.ones(len(group), dtype=bool)\n        last_stop_frame = 0\n        for i, (_, row) in enumerate(group.iterrows()):\n            if row['start_frame'] < last_stop_frame:\n                mask[i] = False\n            else:\n                last_stop_frame = row['stop_frame']\n        group_list.append(group[mask])\n    submission = pd.concat(group_list)\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped duplicate frames\")\n\n    # Rule 3: Submit something for every video (simplified fallback)\n    s_list = []\n    for idx, row in dataset.iterrows():\n        lab_id = row['lab_id']\n        if lab_id.startswith('MABe22'):\n            continue\n        video_id = row['video_id']\n        if (submission.video_id == video_id).any():\n            continue\n\n        if verbose: \n            print(f\"Video {video_id} has no predictions.\")\n        \n        # Simple fallback prediction\n        s_list.append((video_id, 'mouse1', 'self', 'rear', 100, 200))\n\n    if len(s_list) > 0:\n        submission = pd.concat([\n            submission,\n            pd.DataFrame(s_list, columns=['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\n        ])\n        print(\"ERROR: Filled empty videos\")\n\n    submission = submission.reset_index(drop=True)\n    return submission\n\n# ============= MAIN PROCESSING LOOP =============\nprint(\"Starting enhanced processing loop...\")\n\nf1_list = []\nsubmission_list = []\n\nfor section in range(1, len(body_parts_tracked_list)):  # skip index 0 (MABe22)\n    body_parts_tracked_str = body_parts_tracked_list[section]\n    \n    try:\n        body_parts_tracked = json.loads(body_parts_tracked_str)\n        print(f\"\\n{section}. Processing videos with {body_parts_tracked}\")\n    \n        # Read training data for this body parts configuration\n        train_subset = train[train.body_parts_tracked == body_parts_tracked_str]\n        \n        single_mouse_list = []\n        single_mouse_label_list = []\n        single_mouse_meta_list = []\n        mouse_pair_list = []\n        mouse_pair_label_list = []\n        mouse_pair_meta_list = []\n    \n        for switch, data, meta, label in generate_mouse_data(train_subset, 'train'):\n            if switch == 'single':\n                single_mouse_list.append(data)\n                single_mouse_meta_list.append(meta)\n                single_mouse_label_list.append(label)\n            else:\n                mouse_pair_list.append(data)\n                mouse_pair_meta_list.append(meta)\n                mouse_pair_label_list.append(label)\n    \n        # Create enhanced binary classifier\n        binary_classifier = create_simple_ensemble()\n    \n        # Process single-mouse actions\n        if len(single_mouse_list) > 0:\n            # Concatenate all batches\n            single_mouse = pd.concat(single_mouse_list)\n            single_mouse_label = pd.concat(single_mouse_label_list)\n            single_mouse_meta = pd.concat(single_mouse_meta_list)\n            del single_mouse_list, single_mouse_label_list, single_mouse_meta_list\n            \n            # Enhanced feature engineering\n            X_tr = transform_single_enhanced(single_mouse, body_parts_tracked)\n            del single_mouse\n            print(f\"Single mouse features: {X_tr.shape}\")\n    \n            if validate_or_submit == 'validate':\n                cross_validate_classifier_enhanced(binary_classifier, X_tr, single_mouse_label, single_mouse_meta)\n            else:\n                submit_enhanced(body_parts_tracked_str, 'single', binary_classifier, X_tr, single_mouse_label, single_mouse_meta)\n            del X_tr\n                \n        # Process mouse-pair actions  \n        if len(mouse_pair_list) > 0:\n            # Concatenate all batches\n            mouse_pair = pd.concat(mouse_pair_list)\n            mouse_pair_label = pd.concat(mouse_pair_label_list)\n            mouse_pair_meta = pd.concat(mouse_pair_meta_list)\n            del mouse_pair_list, mouse_pair_label_list, mouse_pair_meta_list\n        \n            # Enhanced feature engineering\n            X_tr = transform_pair_enhanced(mouse_pair, body_parts_tracked)\n            del mouse_pair\n            print(f\"Mouse pair features: {X_tr.shape}\")\n    \n            if validate_or_submit == 'validate':\n                cross_validate_classifier_enhanced(binary_classifier, X_tr, mouse_pair_label, mouse_pair_meta)\n            else:\n                submit_enhanced(body_parts_tracked_str, 'pair', binary_classifier, X_tr, mouse_pair_label, mouse_pair_meta)\n            del X_tr\n                \n    except Exception as e:\n        print(f'***Exception*** {e}')\n\n    print()\n\n# ============= FINALIZATION =============\nprint(\"Finalizing submission...\")\n\nif validate_or_submit != 'validate':\n    if len(submission_list) > 0:\n        submission = pd.concat(submission_list)\n    else:\n        # Fallback submission\n        submission = pd.DataFrame({\n            'video_id': [438887472],\n            'agent_id': ['mouse1'],\n            'target_id': ['self'],\n            'action': ['rear'],\n            'start_frame': [278],\n            'stop_frame': [500]\n        })\n    \n    if validate_or_submit == 'submit':\n        submission_robust = robustify(submission, test, 'test')\n    else:\n        submission_robust = robustify(submission, stresstest, 'stresstest', 'stresstest_tracking')\n    \n    submission_robust.index.name = 'row_id'\n    submission_robust.to_csv('submission.csv')\n    \n    print(f\"Final submission shape: {submission_robust.shape}\")\n    print(\"Submission saved to submission.csv\")\n    \n    # Display first few rows\n    print(\"\\nFirst few rows of submission:\")\n    print(submission_robust.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-26T17:40:15.283964Z","iopub.execute_input":"2025-09-26T17:40:15.284239Z","execution_failed":"2025-09-26T17:44:13.428Z"}},"outputs":[],"execution_count":null}]}