{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import accuracy_score, classification_report\nfrom sklearn.model_selection import train_test_split\nfrom collections import defaultdict, Counter\nimport ast\nimport warnings\nwarnings.filterwarnings('ignore')\nimport lightgbm as lgb\ntry:\n    import torch\n    from torch import nn, optim\n    from torch.utils.data import Dataset, DataLoader\n    from torch.cuda.amp import autocast, GradScaler\nexcept ImportError as e:\n    print(f\"Error importing PyTorch: {e}\")\n    raise\nimport pickle\nimport os\nimport psutil\nimport gc\nfrom joblib import Parallel, delayed\n# =============================================================================\n# SHARED UTILITIES: Always load metadata and define functions\n# =============================================================================\nbase_path = '/kaggle/input/MABe-mouse-behavior-detection/'\n# Load metadata\ntrain_meta = pd.read_csv(f'{base_path}train.csv')\ntrain_meta['video_id'] = train_meta['video_id'].astype(str)\ntrain_meta['body_parts_tracked'] = train_meta['body_parts_tracked'].apply(\n    lambda x: ast.literal_eval(x) if pd.notna(x) else []\n)\ntrain_meta['behaviors_labeled'] = train_meta['behaviors_labeled'].apply(\n    lambda x: [b.strip().strip(\"'\\\"\") for b in ast.literal_eval(x)] if pd.notna(x) else []\n)\ntest_meta = pd.read_csv(f'{base_path}test.csv')\ntest_meta['video_id'] = test_meta['video_id'].astype(str)\ntest_meta['body_parts_tracked'] = test_meta['body_parts_tracked'].apply(\n    lambda x: ast.literal_eval(x) if pd.notna(x) else []\n)\ntest_meta['behaviors_labeled'] = test_meta['behaviors_labeled'].apply(\n    lambda x: [b.strip().strip(\"'\\\"\") for b in ast.literal_eval(x)] if pd.notna(x) else []\n)\ndef parse_behavior_label(b_label):\n    \"\"\"Parse 'mouse1,mouse2,approach' to (1,2,'approach').\"\"\"\n    parts = [p.strip().strip(\"'\\\"\") for p in b_label.split(',')]\n    if len(parts) >= 3:\n        agent_str = parts[0].replace('mouse', '').replace('self', '1')\n        agent = int(agent_str)\n        target_str = parts[1].replace('mouse', '').replace('self', str(agent))\n        target = int(target_str)\n        action = parts[2].strip()\n        return agent, target, action\n    return None\ndef load_video_features(lab, vid, is_train=True, N=10):\n    \"\"\"\n    Load tracking; add features. For train: incl. labels; for test: no labels.\n    \"\"\"\n    meta = train_meta if is_train else test_meta\n    row = meta[(meta['lab_id'] == lab) & (meta['video_id'] == vid)].iloc[0]\n    fps = row['frames_per_second']\n    pix_per_cm = row['pix_per_cm_approx']\n    total_frames = int(fps * row['video_duration_sec'])\n    # Load tracking\n    track_dir = 'train_tracking' if is_train else 'test_tracking'\n    track_path = f'{base_path}{track_dir}/{lab}/{vid}.parquet'\n    track = pd.read_parquet(track_path)\n    track['mouse_id'] = track['mouse_id'].astype(int)\n    track = track[track['video_frame'] % N == 0].copy()\n    track['x_cm'] = track['x'] / pix_per_cm\n    track['y_cm'] = track['y'] / pix_per_cm\n    anns = None\n    if is_train:\n        ann_path = f'{base_path}train_annotation/{lab}/{vid}.parquet'\n        anns = pd.read_parquet(ann_path)\n        def expand_to_frames(anns, total_frames):\n            frame_labels = np.full(total_frames, 'background')\n            for _, r in anns.iterrows():\n                action = f\"{int(r['agent_id'])}_{int(r['target_id'])}_{r['action']}\"\n                for f in range(int(r['start_frame']), min(int(r['stop_frame']) + 1, total_frames)):\n                    frame_labels[f] = action\n            return frame_labels\n        frame_labels = expand_to_frames(anns, total_frames)\n        track['label'] = track['video_frame'].apply(\n            lambda f: frame_labels[int(f)] if int(f) < len(frame_labels) else 'background'\n        )\n    # Per-mouse features\n    bc = track[track['bodypart'] == 'body_center'].sort_values(['mouse_id', 'video_frame'])\n    bc['dx'] = bc.groupby('mouse_id')['x_cm'].diff()\n    bc['dy'] = bc.groupby('mouse_id')['y_cm'].diff()\n    bc['speed'] = np.sqrt(bc['dx']**2 + bc['dy']**2)\n    bc['heading'] = np.arctan2(bc['dy'], bc['dx'])\n    bc['dist_center'] = np.sqrt(bc['x_cm']**2 + bc['y_cm']**2)\n    # Pairwise nose distances\n    nose = track[track['bodypart'] == 'nose']\n    pairs = [(1,2), (1,3), (1,4), (2,3), (2,4), (3,4)]\n    pair_dists = defaultdict(list)\n    pair_dists['video_frame'] = []\n    for frame in nose['video_frame'].unique():\n        frame_nose = nose[nose['video_frame'] == frame]\n        pair_dists['video_frame'].append(frame)\n        for a, b in pairs:\n            pos_a = frame_nose[frame_nose['mouse_id'] == a][['x_cm', 'y_cm']].mean()\n            pos_b = frame_nose[frame_nose['mouse_id'] == b][['x_cm', 'y_cm']].mean()\n            dist = np.sqrt((pos_a['x_cm'] - pos_b['x_cm'])**2 + (pos_a['y_cm'] - pos_b['y_cm'])**2) if not pos_a.empty and not pos_b.empty else 0\n            pair_dists[f'dist_{a}_{b}'].append(dist)\n    pair_df = pd.DataFrame(pair_dists)\n    # Merge\n    features_df = bc.merge(pair_df, on='video_frame', how='left').fillna(0)\n    if is_train:\n        features_df['label'] = features_df['video_frame'].apply(\n            lambda f: frame_labels[int(f)] if int(f) < len(frame_labels) else 'background'\n        )\n    return features_df, anns\n# Check for complete run flag\ncomplete_flag = '/kaggle/working/run_complete.flag'\nif os.path.exists(complete_flag):\n    print(\"Previous run complete; skipping training and jumping to inference.\")\nelse:\n    # Training\n    non_mabe_labs = ~train_meta['lab_id'].str.contains('MABe22')\n    train_non_mabe = train_meta[non_mabe_labs]\n    all_behaviors = []\n    for behaviors in train_non_mabe['behaviors_labeled']:\n        all_behaviors.extend(behaviors)\n    action_types = [parse_behavior_label(b)[2] for b in all_behaviors if parse_behavior_label(b)]\n    global_actions = sorted(set(action_types))\n    print(f\"Global actions from non-MABe22: {global_actions}\")\n    print(f\"Number of unique classes: {len(global_actions)}\")\n    print(\"Filtering non-MABe22 train data...\")\n    annotated_non_mabe = train_non_mabe[train_non_mabe['behaviors_labeled'].apply(len) > 0]\n    print(f\"Non-MABe22 annotated videos: {len(annotated_non_mabe)}\")\n    os.makedirs('/kaggle/working/features', exist_ok=True)\n    def log_memory(phase):\n        ram_percent = psutil.virtual_memory().percent\n        print(f\"{phase} - RAM usage: {ram_percent:.2f}%\")\n    log_memory(\"After metadata load\")\n    unique_labels = set(['background'])\n    def process_video(i):\n        row = annotated_non_mabe.iloc[i]\n        lab, vid = row['lab_id'], row['video_id']\n        if lab == 'PleasantMeerkat' and vid == '1375833299':\n            return None, set()\n        feature_path = f'/kaggle/working/features/{lab}_{vid}.pkl'\n        loaded = False\n        if os.path.exists(feature_path):\n            try:\n                with open(feature_path, 'rb') as f:\n                    df = pickle.load(f)\n                print(f\"Loaded cached features for {lab}/{vid}\")\n                loaded = True\n            except (EOFError, pickle.UnpicklingError):\n                print(f\"Error loading cached features for {lab}/{vid}, regenerating...\")\n        if not loaded:\n            df, _ = load_video_features(lab, vid)\n            with open(feature_path, 'wb') as f:\n                pickle.dump(df, f)\n            print(f\"Saved features for {lab}/{vid}\")\n        label_set = set(df['label'].unique()) if 'label' in df.columns else set()\n        return df, label_set\n    print(\"Loading features in parallel...\")\n    results = Parallel(n_jobs=-1)(delayed(process_video)(i) for i in range(len(annotated_non_mabe)))\n    all_dfs = [r[0] for r in results if r[0] is not None]\n    for r in results:\n        unique_labels.update(r[1])\n    log_memory(\"After feature loading\")\n    global_le = LabelEncoder()\n    global_le.fit(sorted(unique_labels))\n    print(f\"Global LE fitted on {len(global_le.classes_)} unique labels: {global_le.classes_}\")\n    all_features = []\n    for i, df in enumerate(all_dfs):\n        if 'label' in df.columns and len(df[df['label'] != 'background']) > 0:\n            df['video_id'] = annotated_non_mabe.iloc[i]['video_id']\n            all_features.append(df)\n    print(f\"Loaded {len(all_features)} videos with non-background labels.\")\n    del all_dfs\n    gc.collect()\n    log_memory(\"After label encoding\")\n    if all_features:\n        feature_cols = ['x_cm', 'y_cm', 'speed', 'heading', 'dist_center', 'dist_1_2', 'dist_1_3', 'dist_1_4', 'dist_2_3', 'dist_2_4', 'dist_3_4', 'mouse_id']\n        batch_size_videos = 20\n        rf = None\n        feat_cols = []\n        mouse_categories = [1, 2, 3, 4]\n        for batch_start in range(0, len(all_features), batch_size_videos):\n            batch_end = min(batch_start + batch_size_videos, len(all_features))\n            batch_features = all_features[batch_start:batch_end]\n            X_list = [df[feature_cols].astype(np.float32) for df in batch_features]\n            X_batch = pd.concat(X_list)\n            X_batch['mouse_id'] = pd.Categorical(X_batch['mouse_id'], categories=mouse_categories)\n            X_batch = pd.get_dummies(X_batch, columns=['mouse_id'], prefix='mouse').fillna(0).astype(np.float32)\n            full_labels = [df['label'].tolist() for df in batch_features]\n            full_labels = [lbl for sublist in full_labels for lbl in sublist]\n            y_batch = np.array([global_le.transform([lbl])[0] for lbl in full_labels], dtype=np.int32)\n            print(f\"Processing training batch {batch_start//batch_size_videos + 1}, samples: {len(y_batch)}\")\n            print(\"Training RF on batch...\")\n            rf = RandomForestClassifier(n_estimators=30, random_state=42, class_weight='balanced', n_jobs=-1, max_depth=10, max_features=0.8)\n            rf.fit(X_batch, y_batch)\n            rf_train_acc = accuracy_score(y_batch, rf.predict(X_batch))\n            print(f\"RF Batch accuracy: {rf_train_acc:.4f}\")\n            if batch_start == 0:\n                feat_cols = X_batch.columns.tolist()\n            del X_batch, y_batch\n            gc.collect()\n            log_memory(f\"After batch {batch_start//batch_size_videos + 1}\")\n        print(\"RF ready.\")\n        log_memory(\"After RF training\")\n        open(complete_flag, 'w').close()\n        print(\"Created run_complete.flag\")\n# =============================================================================\n# INFERENCE ON TEST SET & GENERATE SUBMISSION\n# =============================================================================\nos.makedirs('/kaggle/working/test_features', exist_ok=True)\nsubmission_path = '/kaggle/working/submission.csv'\nsubmission_rows = []\nrow_id = 0\nprocessed_videos = set()\nif os.path.exists(submission_path):\n    existing_sub = pd.read_csv(submission_path)\n    submission_rows = existing_sub.to_dict('records')\n    row_id = existing_sub['row_id'].max() + 1 if not existing_sub.empty else 0\n    processed_videos = set(existing_sub['video_id'].unique())\n    print(f\"Loaded existing submission with {len(submission_rows)} rows. Resuming from row_id {row_id}. Processed videos: {len(processed_videos)}\")\nfeat_cols = feat_cols if 'feat_cols' in locals() else []\nprint(\"Starting inference on test set...\")\nmouse_categories = [1, 2, 3, 4]\nfor i, (idx, test_row) in enumerate(test_meta.iterrows()):\n    lab, vid = test_row['lab_id'], test_row['video_id']\n    if vid in processed_videos:\n        print(f\"Skipping already processed test video {i+1}/{len(test_meta)}: {lab}/{vid}\")\n        continue\n    print(f\"Processing test video: {lab}/{vid}\")\n    test_feature_path = f'/kaggle/working/test_features/{lab}_{vid}.pkl'\n    loaded = False\n    if os.path.exists(test_feature_path):\n        try:\n            with open(test_feature_path, 'rb') as f:\n                df_test = pickle.load(f)\n            print(f\"Loaded cached test features for {lab}/{vid}\")\n            loaded = True\n        except (EOFError, pickle.UnpicklingError):\n            print(f\"Error loading cached test features for {lab}/{vid}, regenerating...\")\n    if not loaded:\n        df_test, _ = load_video_features(lab, vid, is_train=False)\n        with open(test_feature_path, 'wb') as f:\n            pickle.dump(df_test, f)\n        print(f\"Saved test features for {lab}/{vid}\")\n    df_test['video_id'] = vid\n    X_test = df_test[['x_cm', 'y_cm', 'speed', 'heading', 'dist_center', 'dist_1_2', 'dist_1_3', 'dist_1_4', 'dist_2_3', 'dist_2_4', 'dist_3_4', 'mouse_id']].fillna(0).astype(np.float32)\n    X_test['mouse_id'] = pd.Categorical(X_test['mouse_id'], categories=mouse_categories)\n    X_test = pd.get_dummies(X_test, columns=['mouse_id'], prefix='mouse').fillna(0).astype(np.float32)\n    if len(feat_cols) > 0:\n        missing_cols = set(feat_cols) - set(X_test.columns)\n        for col in missing_cols:\n            X_test[col] = 0\n        X_test = X_test[feat_cols].astype(np.float32)\n    rf_probs = rf.predict_proba(X_test)\n    frame_probs = rf_probs\n    frame_preds = np.argmax(frame_probs, axis=1)\n    frame_max_probs = np.max(frame_probs, axis=1)\n    def test_preds_to_intervals(preds, probs, frames, video_behaviors, thresh=0.3):\n        intervals = []\n        global row_id\n        current_action_idx = None\n        start = None\n        for i, (pred, prob, frame) in enumerate(zip(preds, probs, frames)):\n            if prob > thresh and pred != current_action_idx:\n                if current_action_idx is not None:\n                    action_str = global_le.inverse_transform([current_action_idx])[0]\n                    parts = action_str.split('_')\n                    if len(parts) == 3:\n                        agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                        if any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                            intervals.append({\n                                'row_id': row_id,\n                                'video_id': vid,\n                                'agent_id': f'mouse{agent}',\n                                'target_id': f'mouse{target}',\n                                'action': action,\n                                'start_frame': start * 10,\n                                'stop_frame': frame * 10 - 1\n                            })\n                            row_id += 1\n                current_action_idx = pred\n                start = frame\n            elif prob <= thresh and current_action_idx is not None:\n                action_str = global_le.inverse_transform([current_action_idx])[0]\n                parts = action_str.split('_')\n                if len(parts) == 3:\n                    agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                    if any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                        intervals.append({\n                            'row_id': row_id,\n                            'video_id': vid,\n                            'agent_id': f'mouse{agent}',\n                            'target_id': f'mouse{target}',\n                            'action': action,\n                            'start_frame': start * 10,\n                            'stop_frame': frame * 10 - 1\n                        })\n                        row_id += 1\n                current_action_idx = None\n        if current_action_idx is not None:\n            action_str = global_le.inverse_transform([current_action_idx])[0]\n            parts = action_str.split('_')\n            if len(parts) == 3:\n                agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                if any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                    intervals.append({\n                        'row_id': row_id,\n                        'video_id': vid,\n                        'agent_id': f'mouse{agent}',\n                        'target_id': f'mouse{target}',\n                        'action': action,\n                        'start_frame': start * 10,\n                        'stop_frame': frames.iloc[-1] * 10 - 1\n                    })\n                    row_id += 1\n        return intervals\n    video_behaviors = test_row['behaviors_labeled']\n    pred_ints = test_preds_to_intervals(frame_preds, frame_max_probs, df_test['video_frame'], video_behaviors)\n    submission_rows.extend(pred_ints)\n    print(f\"Detected {len(pred_ints)} intervals for {vid}\")\n    del df_test, X_test\n    gc.collect()\n    submission_df = pd.DataFrame(submission_rows)\n    submission_df = submission_df[['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']]\n    submission_df.to_csv(submission_path, index=False)\n    print(f\"Partial submission saved after processing {vid}. Current rows: {len(submission_df)}\")\nprint(f\"Inference completed. Generated {len(submission_rows)} submission rows.\")\nprint(f\"Submission saved: {len(submission_df)} rows in submission.csv\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-11T11:49:10.560804Z","iopub.execute_input":"2025-10-11T11:49:10.561127Z","iopub.status.idle":"2025-10-11T11:55:03.756401Z","shell.execute_reply.started":"2025-10-11T11:49:10.561104Z","shell.execute_reply":"2025-10-11T11:55:03.755622Z"}},"outputs":[],"execution_count":null}]}