{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13719397,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\nMABe Challenge 2025 - Starter Code - Partial dataset\nMulti-Agent Behavior Recognition in Mice\n\nOptimizations for speed:\n1. Process only a subset of training data\n2. Simplified feature extraction\n3. Larger window stride (less overlap)\n4. Single model instead of ensemble\n5. Batch processing with early stopping\n6. Reduced cross-validation folds\n\"\"\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport warnings\nimport gc\nfrom typing import Dict, List, Optional\nfrom dataclasses import dataclass\nfrom tqdm import tqdm\nimport pickle\nimport random\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.metrics import f1_score\n\nfrom xgboost import XGBClassifier\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\n@dataclass\nclass Config:\n    \"\"\"Configuration parameters - optimized for fast execution\"\"\"\n    # Paths\n    data_path: Path = Path('/kaggle/input/MABe-mouse-behavior-detection')\n    output_path: Path = Path('/kaggle/working')\n    \n    # SPEED OPTIMIZATIONS\n    max_train_videos: int = 500  # Process only 500 videos instead of all 8790\n    max_test_videos: Optional[int] = None  # Process all test videos\n    sample_rate: float = 0.3  # Sample 30% of windows from each video\n    \n    # Feature extraction - simplified\n    window_size: int = 60  # Larger windows\n    window_stride: int = 30  # Larger stride = less overlap = faster\n    min_window_fill: float = 0.3  # More lenient\n    use_simple_features: bool = True  # Use only basic features\n    \n    # Model parameters - simplified\n    use_single_model: bool = True  # No ensemble for speed\n    n_splits: int = 2  # Reduced CV folds (was 5)\n    random_state: int = 42\n    \n    # Behavior detection\n    confidence_threshold: float = 0.25\n    min_behavior_duration: int = 5  # Reduced from 10\n    \n    # Processing\n    batch_size: int = 50  # Smaller batches for memory\n    early_stop_batches: int = 10  # Stop after 10 batches if not improving\n\nconfig = Config()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SIMPLIFIED DATA STRUCTURES\n# ============================================================================\n\nclass MouseTrackingData:\n    \"\"\"Simplified container for mouse tracking data\"\"\"\n    \n    def __init__(self, video_id: str, data: pd.DataFrame):\n        self.video_id = video_id\n        self.data = data\n        self._preprocess()\n    \n    def _preprocess(self):\n        \"\"\"Minimal preprocessing\"\"\"\n        self.frame_col = None\n        for col in ['video_frame', 'frame', 'frame_number']:\n            if col in self.data.columns:\n                self.frame_col = col\n                break\n        \n        if self.frame_col is None:\n            self.data['frame'] = self.data.index\n            self.frame_col = 'frame'\n    \n    def get_trajectory(self) -> pd.DataFrame:\n        \"\"\"Get simplified trajectory - just centroid\"\"\"\n        if 'x' in self.data.columns and 'y' in self.data.columns:\n            trajectory = self.data.groupby(self.frame_col).agg({\n                'x': 'mean',\n                'y': 'mean'\n            }).reset_index()\n            trajectory.columns = ['frame', 'x', 'y']\n            return trajectory\n        return pd.DataFrame()\n\n# ============================================================================\n# FAST FEATURE EXTRACTION\n# ============================================================================\n\nclass FastFeatureExtractor:\n    \"\"\"Simplified feature extraction for speed\"\"\"\n    \n    def __init__(self, window_size: int = 60, stride: int = 30):\n        self.window_size = window_size\n        self.stride = stride\n    \n    def extract_basic_features(self, trajectory: pd.DataFrame) -> Dict[str, float]:\n        \"\"\"Extract only essential movement features\"\"\"\n        features = {}\n        \n        if len(trajectory) < 2:\n            return features\n        \n        x = trajectory['x'].values\n        y = trajectory['y'].values\n        \n        dx = np.diff(x)\n        dy = np.diff(y)\n        velocity = np.sqrt(dx**2 + dy**2)\n        \n        features['vel_mean'] = np.mean(velocity)\n        features['vel_std'] = np.std(velocity)\n        features['vel_max'] = np.max(velocity)\n        features['total_dist'] = np.sum(velocity)\n        features['net_disp'] = np.sqrt((x[-1] - x[0])**2 + (y[-1] - y[0])**2)\n        features['x_range'] = np.max(x) - np.min(x)\n        features['y_range'] = np.max(y) - np.min(y)\n        features['stationary'] = np.mean(velocity < 2.0)\n        features['fast_move'] = np.mean(velocity > 20.0)\n        \n        return features\n    \n    def extract_windows_fast(self, tracking_data: MouseTrackingData, \n                            sample_rate: float = 1.0) -> pd.DataFrame:\n        \"\"\"Fast window extraction with sampling\"\"\"\n        trajectory = tracking_data.get_trajectory()\n        \n        if trajectory.empty or len(trajectory) < self.window_size:\n            return pd.DataFrame()\n        \n        all_features = []\n        min_frame = trajectory['frame'].min()\n        max_frame = trajectory['frame'].max()\n        \n        window_starts = list(range(int(min_frame), \n                                  int(max_frame) - self.window_size + 1, \n                                  self.stride))\n        \n        if sample_rate < 1.0:\n            n_samples = max(1, int(len(window_starts) * sample_rate))\n            window_starts = random.sample(window_starts, min(n_samples, len(window_starts)))\n        \n        for start_frame in window_starts:\n            end_frame = start_frame + self.window_size\n            window = trajectory[(trajectory['frame'] >= start_frame) & \n                               (trajectory['frame'] < end_frame)]\n            \n            if len(window) < self.window_size * config.min_window_fill:\n                continue\n            \n            features = {\n                'video_id': tracking_data.video_id,\n                'start_frame': start_frame,\n                'end_frame': end_frame,\n            }\n            \n            basic_feats = self.extract_basic_features(window)\n            features.update(basic_feats)\n            \n            all_features.append(features)\n        \n        return pd.DataFrame(all_features)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# SIMPLIFIED ANNOTATION PROCESSING\n# ============================================================================\n\nclass FastAnnotationProcessor:\n    \"\"\"Fast annotation processing\"\"\"\n    \n    @staticmethod\n    def load_annotations(annotation_path: Path) -> pd.DataFrame:\n        \"\"\"Load annotations with minimal processing\"\"\"\n        if not annotation_path.exists():\n            return pd.DataFrame()\n        \n        try:\n            annotations = pd.read_parquet(annotation_path)\n            col_mapping = {\n                'start': 'start_frame',\n                'end': 'end_frame',\n                'stop': 'end_frame',\n                'stop_frame': 'end_frame'\n            }\n            \n            for old_col, new_col in col_mapping.items():\n                if old_col in annotations.columns and new_col not in annotations.columns:\n                    annotations[new_col] = annotations[old_col]\n            \n            if 'start_frame' in annotations.columns and 'end_frame' in annotations.columns:\n                return annotations[['start_frame', 'end_frame']]\n            \n            return pd.DataFrame()\n            \n        except:\n            return pd.DataFrame()\n    \n    @staticmethod\n    def create_labels_fast(features: pd.DataFrame, \n                          annotations: pd.DataFrame) -> np.ndarray:\n        \"\"\"Fast label creation\"\"\"\n        if annotations.empty or 'start_frame' not in annotations.columns:\n            return np.zeros(len(features))\n        \n        labels = np.zeros(len(features))\n        for idx, window in features.iterrows():\n            overlaps = ((annotations['start_frame'] <= window['end_frame']) & \n                       (annotations['end_frame'] >= window['start_frame']))\n            labels[idx] = 1 if overlaps.any() else 0\n        \n        return labels","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# FAST MODEL (XGBoost)\n# ============================================================================\n\nclass FastBehaviorDetector:\n    \"\"\"Simplified model for fast training with XGBoost\"\"\"\n    \n    def __init__(self):\n        self.model = None\n        self.scaler = None\n        self.feature_columns = None\n    \n    def prepare_features(self, features_df: pd.DataFrame) -> np.ndarray:\n        \"\"\"Prepare features quickly\"\"\"\n        numeric_cols = features_df.select_dtypes(include=[np.number]).columns\n        exclude = ['start_frame', 'end_frame']\n        self.feature_columns = [c for c in numeric_cols if c not in exclude]\n        \n        if not self.feature_columns:\n            return np.zeros((len(features_df), 1))\n        \n        X = features_df[self.feature_columns].fillna(0).values\n        return X\n    \n    def train_fast(self, X: np.ndarray, y: np.ndarray):\n        \"\"\"Fast training with single XGBoost model\"\"\"\n        print(f\"Fast training on {X.shape[0]} samples, {X.shape[1]} features\")\n        print(f\"Positive rate: {100*np.mean(y):.1f}%\")\n        \n        if X.shape[0] == 0 or X.shape[1] == 0:\n            return\n        \n        X_train, X_val, y_train, y_val = train_test_split(\n            X, y, test_size=0.2, random_state=config.random_state, stratify=y\n        )\n        \n        self.scaler = StandardScaler()\n        X_train = self.scaler.fit_transform(X_train)\n        X_val = self.scaler.transform(X_val)\n        \n        self.model = XGBClassifier(\n            n_estimators=100,\n            max_depth=6,\n            learning_rate=0.1,\n            subsample=0.8,\n            colsample_bytree=0.8,\n            reg_lambda=1.0,\n            min_child_weight=5,\n            random_state=config.random_state,\n            n_jobs=-1,\n            tree_method=\"hist\",\n            eval_metric=\"logloss\",\n            verbosity=0\n        )\n        \n        self.model.fit(\n            X_train, y_train,\n            eval_set=[(X_val, y_val)],\n            early_stopping_rounds=30,\n            verbose=False\n        )\n        \n        val_pred = self.model.predict(X_val)\n        val_f1 = f1_score(y_val, val_pred)\n        print(f\"Validation F1: {val_f1:.3f}\")\n    \n    def predict_fast(self, X: np.ndarray) -> np.ndarray:\n        \"\"\"Fast prediction\"\"\"\n        if self.model is None:\n            return np.zeros(X.shape[0])\n        \n        X_scaled = self.scaler.transform(X)\n        return self.model.predict_proba(X_scaled)[:, 1]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# MAIN FAST PIPELINE\n# ============================================================================\n\nclass FastMABePipeline:\n    \"\"\"Optimized pipeline for speed\"\"\"\n    \n    def __init__(self):\n        self.feature_extractor = FastFeatureExtractor(\n            window_size=config.window_size,\n            stride=config.window_stride\n        )\n        self.annotation_processor = FastAnnotationProcessor()\n        self.detector = FastBehaviorDetector()\n    \n    def process_training_fast(self):\n        \"\"\"Fast training data processing\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"FAST TRAINING DATA PROCESSING\")\n        print(\"=\"*60)\n        \n        tracking_files = []\n        train_path = config.data_path / 'train_tracking'\n        \n        if train_path.exists():\n            for lab_dir in train_path.iterdir():\n                if lab_dir.is_dir():\n                    files = list(lab_dir.glob('*.parquet'))\n                    tracking_files.extend(files)\n        \n        if config.max_train_videos and len(tracking_files) > config.max_train_videos:\n            print(f\"Sampling {config.max_train_videos} from {len(tracking_files)} files\")\n            tracking_files = random.sample(tracking_files, config.max_train_videos)\n        \n        print(f\"Processing {len(tracking_files)} files\")\n        \n        all_features = []\n        all_labels = []\n        \n        for i in range(0, len(tracking_files), config.batch_size):\n            batch = tracking_files[i:i+config.batch_size]\n            batch_num = i//config.batch_size + 1\n            total_batches = (len(tracking_files)-1)//config.batch_size + 1\n            \n            print(f\"\\nBatch {batch_num}/{total_batches}\")\n            \n            for file_path in tqdm(batch, desc=\"Processing\"):\n                try:\n                    data = pd.read_parquet(file_path)\n                    tracking = MouseTrackingData(file_path.stem, data)\n                    features = self.feature_extractor.extract_windows_fast(\n                        tracking, sample_rate=config.sample_rate\n                    )\n                    \n                    if features.empty:\n                        continue\n                    \n                    ann_path = config.data_path / 'train_annotation' / \\\n                              file_path.parent.name / f\"{file_path.stem}.parquet\"\n                    annotations = self.annotation_processor.load_annotations(ann_path)\n                    \n                    labels = self.annotation_processor.create_labels_fast(\n                        features, annotations\n                    )\n                    \n                    all_features.append(features)\n                    all_labels.append(labels)\n                    \n                except Exception as e:\n                    continue\n            \n            if len(all_features) > 0:\n                total_samples = sum(len(f) for f in all_features)\n                if total_samples > 50000:\n                    print(f\"Early stop: {total_samples} samples collected\")\n                    break\n            \n            gc.collect()\n        \n        if all_features:\n            features_df = pd.concat(all_features, ignore_index=True)\n            labels = np.concatenate(all_labels)\n            print(f\"\\nTotal samples: {len(features_df)}\")\n            print(f\"Positive rate: {100*np.mean(labels):.1f}%\")\n            return features_df, labels\n        \n        return None, None\n    \n    def train_model_fast(self, features_df: pd.DataFrame, labels: np.ndarray):\n        \"\"\"Fast model training\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"FAST MODEL TRAINING\")\n        print(\"=\"*60)\n        \n        X = self.detector.prepare_features(features_df)\n        self.detector.train_fast(X, labels)\n    \n    def generate_predictions_fast(self) -> pd.DataFrame:\n        \"\"\"Fast prediction generation\"\"\"\n        print(\"\\n\" + \"=\"*60)\n        print(\"FAST PREDICTION GENERATION\")\n        print(\"=\"*60)\n        \n        test_path = config.data_path / 'test_tracking'\n        test_files = []\n        \n        if test_path.exists():\n            for lab_dir in test_path.iterdir():\n                if lab_dir.is_dir():\n                    test_files.extend(list(lab_dir.glob('*.parquet')))\n        \n        print(f\"Found {len(test_files)} test files\")\n        \n        if config.max_test_videos and len(test_files) > config.max_test_videos:\n            test_files = test_files[:config.max_test_videos]\n        \n        predictions = []\n        \n        for test_file in tqdm(test_files, desc=\"Predicting\"):\n            try:\n                data = pd.read_parquet(test_file)\n                tracking = MouseTrackingData(test_file.stem, data)\n                features = self.feature_extractor.extract_windows_fast(\n                    tracking, sample_rate=1.0\n                )\n                \n                if features.empty:\n                    continue\n                \n                X = self.detector.prepare_features(features)\n                proba = self.detector.predict_fast(X)\n                \n                for idx, prob in enumerate(proba):\n                    if prob > config.confidence_threshold:\n                        predictions.append({\n                            'video_id': features.iloc[idx]['video_id'],\n                            'agent_id': 'mouse1',\n                            'target_id': 'mouse2',\n                            'action': 'sniff',\n                            'start_frame': int(features.iloc[idx]['start_frame']),\n                            'stop_frame': int(features.iloc[idx]['end_frame'])\n                        })\n            except:\n                continue\n        \n        if predictions:\n            pred_df = pd.DataFrame(predictions)\n            pred_df = pred_df.drop_duplicates()\n            pred_df = pred_df.sort_values(['video_id', 'start_frame'])\n            return pred_df\n        else:\n            return pd.DataFrame({\n                'video_id': ['test_video'],\n                'agent_id': ['mouse1'],\n                'target_id': ['mouse2'],\n                'action': ['grooming'],\n                'start_frame': [0],\n                'stop_frame': [30]\n            })\n    \n    def create_submission(self, predictions: pd.DataFrame) -> pd.DataFrame:\n        \"\"\"Create submission file\"\"\"\n        predictions['row_id'] = range(len(predictions))\n        submission = predictions[['row_id', 'video_id', 'agent_id', 'target_id', \n                                 'action', 'start_frame', 'stop_frame']]\n        \n        submission.to_csv('submission.csv', index=False)\n        print(f\"\\nCreated submission.csv with {len(submission)} predictions\")\n        return submission","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================================\n# MAIN EXECUTION\n# ============================================================================\n\ndef main():\n    \"\"\"Main execution - optimized for speed\"\"\"\n    import time\n    start_time = time.time()\n    \n    print(\"\\n\" + \"=\"*80)\n    print(\"MABe CHALLENGE 2025 - FAST STARTER CODE (XGBoost Version)\")\n    print(\"Optimized to run in < 12 hours\")\n    print(\"=\"*80)\n    \n    print(f\"\\nSpeed Settings:\")\n    print(f\"  Max training videos: {config.max_train_videos}\")\n    print(f\"  Window sampling rate: {config.sample_rate}\")\n    print(f\"  Window size: {config.window_size}\")\n    print(f\"  Window stride: {config.window_stride}\")\n    \n    pipeline = FastMABePipeline()\n    \n    features_df, labels = pipeline.process_training_fast()\n    \n    if features_df is not None and len(features_df) > 100:\n        pipeline.train_model_fast(features_df, labels)\n        \n        with open('fast_model.pkl', 'wb') as f:\n            pickle.dump(pipeline.detector, f)\n        print(\"Model saved to fast_model.pkl\")\n        \n        predictions = pipeline.generate_predictions_fast()\n        submission = pipeline.create_submission(predictions)\n        \n        elapsed = time.time() - start_time\n        hours = elapsed / 3600\n        print(f\"\\nTotal runtime: {hours:.2f} hours\")\n        \n        print(\"\\n\" + \"=\"*80)\n        print(\"PIPELINE COMPLETE!\")\n        print(\"=\"*80)\n        \n        return submission\n    else:\n        print(\"\\nInsufficient training data!\")\n        return None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    submission = main()\n    if submission is not None:\n        print(\"\\n✓ Execution completed successfully!\")\n        print(f\"Submission preview:\\n{submission.head()}\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}