{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport gc\nfrom pathlib import Path\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import f1_score, classification_report, confusion_matrix\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Fix for PyTorch 2.6 weights_only issue\ntry:\n    from torch.serialization import add_safe_globals\n    add_safe_globals([np.core.multiarray.scalar])\nexcept:\n    pass\n\nclass MouseDataset:\n    \"\"\"Mouse dataset handler\"\"\"\n    \n    def __init__(self, dataset_path):\n        self.dataset_path = Path(dataset_path)\n        self.tracking_files = []\n        \n    def discover_files(self):\n        \"\"\"Discover dataset files\"\"\"\n        print(\"🔍 DISCOVERING FILES...\")\n        \n        tracking_path = self.dataset_path / 'train_tracking'\n        \n        for lab_folder in tracking_path.iterdir():\n            if lab_folder.is_dir():\n                parquet_files = list(lab_folder.glob(\"*.parquet\"))\n                self.tracking_files.extend(parquet_files)\n        \n        print(f\"📁 Found {len(self.tracking_files)} tracking files\")\n        return self.tracking_files\n\nclass FeatureEngineer:\n    \"\"\"Feature engineering with proper mouse handling\"\"\"\n    \n    def __init__(self):\n        self.feature_cache = {}\n        \n    def compute_features(self, video_data):\n        \"\"\"Compute features for mouse behavior\"\"\"\n        if video_data.empty:\n            return None\n            \n        features = {}\n        \n        # Sort by frame\n        video_data = video_data.sort_values('video_frame')\n        \n        # Get ALL mice from the data\n        all_mice = sorted(video_data['mouse_id'].unique())\n        features['total_mice'] = len(all_mice)\n        \n        print(f\"   Detected mice: {all_mice}\")  # Debug info\n        \n        # Features for each mouse\n        for mouse_id in all_mice:\n            mouse_data = video_data[video_data['mouse_id'] == mouse_id]\n            \n            if len(mouse_data) > 1:\n                mouse_data = mouse_data.sort_values('video_frame')\n                x_pos = mouse_data['x'].values\n                y_pos = mouse_data['y'].values\n                \n                # Movement features\n                if len(x_pos) > 1:\n                    dx = np.diff(x_pos)\n                    dy = np.diff(y_pos)\n                    \n                    speed = np.sqrt(dx**2 + dy**2)\n                    \n                    features[f'mouse_{mouse_id}_speed_mean'] = np.mean(speed)\n                    features[f'mouse_{mouse_id}_speed_max'] = np.max(speed)\n                    features[f'mouse_{mouse_id}_x_mean'] = np.mean(x_pos)\n                    features[f'mouse_{mouse_id}_y_mean'] = np.mean(y_pos)\n                    \n                    # Distance traveled\n                    total_distance = np.sum(np.sqrt(dx**2 + dy**2))\n                    features[f'mouse_{mouse_id}_distance'] = total_distance\n        \n        # Social interactions between ALL mice\n        if len(all_mice) >= 2:\n            for i in range(len(all_mice)):\n                for j in range(i+1, len(all_mice)):\n                    mouse1, mouse2 = all_mice[i], all_mice[j]\n                    social_features = self._compute_social_interaction(video_data, mouse1, mouse2)\n                    features.update(social_features)\n        \n        return features\n    \n    def _compute_social_interaction(self, video_data, mouse1, mouse2):\n        \"\"\"Compute social interaction features\"\"\"\n        features = {}\n        \n        m1_data = video_data[video_data['mouse_id'] == mouse1].sort_values('video_frame')\n        m2_data = video_data[video_data['mouse_id'] == mouse2].sort_values('video_frame')\n        \n        if len(m1_data) > 0 and len(m2_data) > 0:\n            common_frames = set(m1_data['video_frame']).intersection(set(m2_data['video_frame']))\n            if len(common_frames) > 0:\n                distances = []\n                for frame in sorted(common_frames)[:50]:\n                    m1_frame = m1_data[m1_data['video_frame'] == frame]\n                    m2_frame = m2_data[m2_data['video_frame'] == frame]\n                    \n                    if len(m1_frame) > 0 and len(m2_frame) > 0:\n                        dist = np.sqrt(\n                            (m1_frame['x'].mean() - m2_frame['x'].mean())**2 +\n                            (m1_frame['y'].mean() - m2_frame['y'].mean())**2\n                        )\n                        distances.append(dist)\n                \n                if distances:\n                    features[f'dist_{mouse1}_{mouse2}_mean'] = np.mean(distances)\n                    features[f'dist_{mouse1}_{mouse2}_min'] = np.min(distances)\n        \n        return features\n\nclass MouseBehaviorDataset(Dataset):\n    \"\"\"PyTorch Dataset for mouse behavior\"\"\"\n    \n    def __init__(self, features, labels):\n        self.features = torch.FloatTensor(features)\n        self.labels = torch.LongTensor(labels)\n        \n    def __len__(self):\n        return len(self.features)\n    \n    def __getitem__(self, idx):\n        return self.features[idx], self.labels[idx]\n\nclass MouseBehaviorNN(nn.Module):\n    \"\"\"Neural Network for mouse behavior classification\"\"\"\n    \n    def __init__(self, input_size, num_classes):\n        super(MouseBehaviorNN, self).__init__()\n        \n        self.network = nn.Sequential(\n            nn.Linear(input_size, 256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            \n            nn.Linear(256, 128),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            \n            nn.Linear(128, 64),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            \n            nn.Linear(64, num_classes)\n        )\n        \n    def forward(self, x):\n        return self.network(x)\n\nclass MouseBehaviorPipeline:\n    \"\"\"Complete pipeline for mouse behavior detection\"\"\"\n    \n    def __init__(self, dataset_path):\n        self.dataset_path = Path(dataset_path)\n        self.dataset_handler = MouseDataset(dataset_path)\n        self.feature_engineer = FeatureEngineer()\n        self.scaler = StandardScaler()\n        self.label_encoder = LabelEncoder()\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        print(f\"🚀 Using device: {self.device}\")\n        \n    def process_training_data(self):\n        \"\"\"Process training data\"\"\"\n        print(\"🎯 PROCESSING TRAINING DATA...\")\n        \n        tracking_files = self.dataset_handler.discover_files()\n        \n        all_features = []\n        all_labels = []\n        \n        for file_path in tqdm(tracking_files[:2000], desc=\"Processing videos\"):\n            try:\n                df = pd.read_parquet(file_path)\n                \n                # Sample frames for efficiency\n                if len(df) > 1000:\n                    df = df.iloc[::10].copy()\n                \n                # Compute features\n                lab = file_path.parent.name\n                features = self.feature_engineer.compute_features(df)\n                \n                if features:\n                    # Map lab to behavior\n                    lab_behavior_map = {\n                        'NiftyGoldfinch': 'sniff',\n                        'GroovyShrew': 'rear', \n                        'CalMS21_supplemental': 'sniff',\n                        'CalMS21_task1': 'sniffgenital',\n                        'CalMS21_task2': 'attack',\n                        'AdaptableSnail': 'attack',\n                        'SparklingTapir': 'attack',\n                        'CRIM13': 'approach',\n                        'UppityFerret': 'sniffgenital',\n                    }\n                    label = lab_behavior_map.get(lab, 'sniff')\n                    \n                    all_features.append(features)\n                    all_labels.append(label)\n                \n                del df\n                \n            except Exception as e:\n                continue\n        \n        print(f\"✅ Processed {len(all_features)} training samples\")\n        return all_features, all_labels\n    \n    def prepare_features(self, all_features, all_labels):\n        \"\"\"Prepare features for training\"\"\"\n        print(\"🛠️ PREPARING FEATURES...\")\n        \n        # Convert to DataFrame\n        features_df = pd.DataFrame(all_features).fillna(0)\n        features_df = features_df.replace([np.inf, -np.inf], 0)\n        \n        # Get feature columns\n        feature_columns = [col for col in features_df.columns if col != 'total_mice']\n        X = self.scaler.fit_transform(features_df[feature_columns])\n        y = self.label_encoder.fit_transform(all_labels)\n        \n        print(f\"📊 Dataset: {X.shape[0]} samples, {X.shape[1]} features\")\n        print(f\"🎯 Behaviors: {list(self.label_encoder.classes_)}\")\n        \n        return X, y, feature_columns\n    \n    def train_model(self, X, y):\n        \"\"\"Train the model\"\"\"\n        print(\"🧠 TRAINING MODEL...\")\n        \n        X_train, X_val, y_train, y_val = train_test_split(\n            X, y, test_size=0.2, random_state=42, stratify=y\n        )\n        \n        print(f\"📊 Training: {X_train.shape[0]}, Validation: {X_val.shape[0]}\")\n        \n        train_dataset = MouseBehaviorDataset(X_train, y_train)\n        val_dataset = MouseBehaviorDataset(X_val, y_val)\n        \n        train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n        val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n        \n        model = MouseBehaviorNN(X.shape[1], len(np.unique(y))).to(self.device)\n        criterion = nn.CrossEntropyLoss()\n        optimizer = optim.Adam(model.parameters(), lr=0.001)\n        \n        best_val_f1 = 0\n        best_model = None\n        \n        for epoch in range(50):\n            # Training\n            model.train()\n            train_loss = 0\n            for batch_features, batch_labels in train_loader:\n                batch_features = batch_features.to(self.device)\n                batch_labels = batch_labels.to(self.device)\n                \n                optimizer.zero_grad()\n                outputs = model(batch_features)\n                loss = criterion(outputs, batch_labels)\n                loss.backward()\n                optimizer.step()\n                train_loss += loss.item()\n            \n            # Validation\n            model.eval()\n            all_preds = []\n            all_labels = []\n            with torch.no_grad():\n                for batch_features, batch_labels in val_loader:\n                    batch_features = batch_features.to(self.device)\n                    batch_labels = batch_labels.to(self.device)\n                    \n                    outputs = model(batch_features)\n                    _, preds = torch.max(outputs, 1)\n                    all_preds.extend(preds.cpu().numpy())\n                    all_labels.extend(batch_labels.cpu().numpy())\n            \n            val_f1 = f1_score(all_labels, all_preds, average='weighted')\n            \n            if val_f1 > best_val_f1:\n                best_val_f1 = val_f1\n                best_model = model.state_dict().copy()\n            \n            if (epoch + 1) % 10 == 0:\n                print(f\"Epoch {epoch+1}/50: Train Loss: {train_loss/len(train_loader):.4f}, Val F1: {val_f1:.4f}\")\n        \n        if best_model:\n            model.load_state_dict(best_model)\n            torch.save(model.state_dict(), 'best_mouse_model.pth')\n        \n        print(f\"🎯 Best Validation F1: {best_val_f1:.4f}\")\n        return model\n    \n    def create_submission(self, model, feature_columns):\n        \"\"\"Create submission.csv with proper format\"\"\"\n        print(\"📤 CREATING SUBMISSION.CSV...\")\n        \n        test_path = self.dataset_path / 'test_tracking'\n        if not test_path.exists():\n            print(\"❌ Test data not found!\")\n            return None\n        \n        test_files = list(test_path.rglob(\"*.parquet\"))\n        print(f\"🎯 Found {len(test_files)} test videos\")\n        \n        all_predictions = []\n        \n        for file_path in test_files:\n            try:\n                df = pd.read_parquet(file_path)\n                video_id = file_path.stem  # Just the filename\n                \n                # Get ALL mice from test data\n                all_mice = sorted(df['mouse_id'].unique())\n                print(f\"   Video {video_id}: Mice detected - {all_mice}\")\n                \n                # Process the entire video or in segments\n                total_frames = df['video_frame'].max()\n                \n                # Create multiple segments\n                for segment_num in range(5):\n                    start_frame = segment_num * 2000\n                    end_frame = min((segment_num + 1) * 2000, total_frames)\n                    \n                    if end_frame - start_frame < 500:\n                        continue\n                    \n                    segment_data = df[(df['video_frame'] >= start_frame) & \n                                     (df['video_frame'] < end_frame)]\n                    \n                    if len(segment_data) < 100:\n                        continue\n                    \n                    # Sample frames\n                    if len(segment_data) > 500:\n                        segment_data = segment_data.iloc[::5].copy()\n                    \n                    # Compute features\n                    features = self.feature_engineer.compute_features(segment_data)\n                    \n                    if features:\n                        # Prepare for prediction\n                        features_df = pd.DataFrame([features]).fillna(0)\n                        features_df = features_df.replace([np.inf, -np.inf], 0)\n                        \n                        # Ensure all feature columns exist\n                        for col in feature_columns:\n                            if col not in features_df.columns:\n                                features_df[col] = 0\n                        \n                        X_test = self.scaler.transform(features_df[feature_columns])\n                        X_test_tensor = torch.FloatTensor(X_test).to(self.device)\n                        \n                        # Predict\n                        model.eval()\n                        with torch.no_grad():\n                            outputs = model(X_test_tensor)\n                            _, prediction = torch.max(outputs, 1)\n                            behavior = self.label_encoder.inverse_transform(prediction.cpu().numpy())[0]\n                        \n                        # Create predictions for ALL mouse pairs\n                        for i in range(len(all_mice)):\n                            for j in range(len(all_mice)):\n                                if i != j:  # Different mice\n                                    agent_id = f\"mouse{all_mice[i]}\"\n                                    target_id = f\"mouse{all_mice[j]}\"\n                                    \n                                    all_predictions.append({\n                                        'row_id': len(all_predictions),\n                                        'video_id': video_id,\n                                        'agent_id': agent_id,\n                                        'target_id': target_id,\n                                        'action': behavior,\n                                        'start_frame': int(start_frame),\n                                        'stop_frame': int(end_frame)\n                                    })\n                \n                del df\n                gc.collect()\n                \n            except Exception as e:\n                print(f\"⚠️ Error processing {file_path}: {e}\")\n                continue\n        \n        # Create DataFrame with correct column order\n        submission_df = pd.DataFrame(all_predictions)\n        \n        # Ensure correct column order\n        submission_df = submission_df[['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']]\n        \n        # Save to CSV\n        submission_path = '/kaggle/working/submission.csv'\n        submission_df.to_csv(submission_path, index=False)\n        \n        print(f\"✅ SUBMISSION.CSV CREATED: {submission_df.shape}\")\n        print(f\"📁 Saved: {submission_path}\")\n        \n        print(\"\\n📊 Submission preview:\")\n        print(submission_df.head())\n        \n        print(\"\\n📈 Behavior distribution:\")\n        print(submission_df['action'].value_counts())\n        \n        print(f\"\\n🐭 Unique agent IDs: {sorted(submission_df['agent_id'].unique())}\")\n        print(f\"🐭 Unique target IDs: {sorted(submission_df['target_id'].unique())}\")\n        \n        return submission_df\n\n    def run_complete_pipeline(self):\n        \"\"\"Run the complete pipeline\"\"\"\n        print(\"=\" * 80)\n        print(\"🚀 MOUSE BEHAVIOR DETECTION PIPELINE\")\n        print(\"🎯 COMPLETE TRAINING + SUBMISSION\")\n        print(\"=\" * 80)\n        \n        try:\n            # Step 1: Process training data\n            print(\"\\n1. 📥 PROCESSING TRAINING DATA\")\n            all_features, all_labels = self.process_training_data()\n            \n            if not all_features:\n                print(\"❌ No training data processed!\")\n                return 0\n            \n            # Step 2: Prepare features\n            print(\"\\n2. 🛠️ PREPARING FEATURES\")\n            X, y, feature_columns = self.prepare_features(all_features, all_labels)\n            \n            # Step 3: Train model\n            print(\"\\n3. 🧠 TRAINING MODEL\")\n            model = self.train_model(X, y)\n            \n            # Step 4: Create submission\n            print(\"\\n4. 📤 CREATING SUBMISSION\")\n            submission = self.create_submission(model, feature_columns)\n            \n            print(\"\\n\" + \"=\" * 80)\n            print(\"🎉 PIPELINE COMPLETED SUCCESSFULLY!\")\n            \n            if submission is not None:\n                print(\"📁 Submission saved: /kaggle/working/submission.csv\")\n                print(f\"📊 Total predictions: {len(submission)}\")\n            \n            print(\"=\" * 80)\n            return 1\n            \n        except Exception as e:\n            print(f\"❌ Pipeline failed: {e}\")\n            import traceback\n            traceback.print_exc()\n            return 0\n\n# Run the pipeline\nif __name__ == \"__main__\":\n    DATASET_PATH = \"/kaggle/input/MABe-mouse-behavior-detection\"\n    \n    print(\"🚀 STARTING MOUSE BEHAVIOR PIPELINE\")\n    print(\"=\" * 80)\n    \n    pipeline = MouseBehaviorPipeline(DATASET_PATH)\n    success = pipeline.run_complete_pipeline()\n    \n    if success:\n        print(\"✅ PIPELINE COMPLETED SUCCESSFULLY!\")\n        print(\"🎯 Your submission.csv is ready!\")\n    else:\n        print(\"❌ Pipeline failed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T12:48:19.834069Z","iopub.execute_input":"2025-10-25T12:48:19.834674Z","iopub.status.idle":"2025-10-25T12:52:29.132885Z","shell.execute_reply.started":"2025-10-25T12:48:19.834646Z","shell.execute_reply":"2025-10-25T12:52:29.132104Z"}},"outputs":[],"execution_count":null}]}