{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score\nfrom collections import defaultdict, Counter\nimport ast\nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc\nfrom joblib import Parallel, delayed\nimport psutil\nimport os\n\n# =============================================================================\n# SHARED UTILITIES: Always load metadata and define functions\n# =============================================================================\nbase_path = '/kaggle/input/MABe-mouse-behavior-detection/'\n# Load metadata\ntrain_meta = pd.read_csv(f'{base_path}train.csv')\ntrain_meta['video_id'] = train_meta['video_id'].astype(str)\ntrain_meta['body_parts_tracked'] = train_meta['body_parts_tracked'].apply(\n    lambda x: ast.literal_eval(x) if pd.notna(x) else []\n)\ntrain_meta['behaviors_labeled'] = train_meta['behaviors_labeled'].apply(\n    lambda x: [b.strip().strip(\"'\\\"\") for b in ast.literal_eval(x)] if pd.notna(x) else []\n)\ntest_meta = pd.read_csv(f'{base_path}test.csv')\ntest_meta['video_id'] = test_meta['video_id'].astype(str)\ntest_meta['body_parts_tracked'] = test_meta['body_parts_tracked'].apply(\n    lambda x: ast.literal_eval(x) if pd.notna(x) else []\n)\ntest_meta['behaviors_labeled'] = test_meta['behaviors_labeled'].apply(\n    lambda x: [b.strip().strip(\"'\\\"\") for b in ast.literal_eval(x)] if pd.notna(x) else []\n)\ndef parse_behavior_label(b_label):\n    \"\"\"Parse 'mouse1,mouse2,approach' to (1,2,'approach').\"\"\"\n    parts = [p.strip().strip(\"'\\\"\") for p in b_label.split(',')]\n    if len(parts) >= 3:\n        agent_str = parts[0].replace('mouse', '').replace('self', '1')\n        agent = int(agent_str)\n        target_str = parts[1].replace('mouse', '').replace('self', str(agent))\n        target = int(target_str)\n        action = parts[2].strip()\n        return agent, target, action\n    return None\ndef load_video_features(lab, vid, is_train=True, N=10):\n    \"\"\"\n    Load tracking; add features. For train: incl. labels; for test: no labels.\n    \"\"\"\n    meta = train_meta if is_train else test_meta\n    row = meta[(meta['lab_id'] == lab) & (meta['video_id'] == vid)].iloc[0]\n    fps = row['frames_per_second']\n    pix_per_cm = row['pix_per_cm_approx']\n    total_frames = int(fps * row['video_duration_sec'])\n    # Load tracking\n    track_dir = 'train_tracking' if is_train else 'test_tracking'\n    track_path = f'{base_path}{track_dir}/{lab}/{vid}.parquet'\n    track = pd.read_parquet(track_path)\n    track['mouse_id'] = track['mouse_id'].astype(int)\n    track = track[track['video_frame'] % N == 0].copy()\n    track['x_cm'] = track['x'] / pix_per_cm\n    track['y_cm'] = track['y'] / pix_per_cm\n    anns = None\n    if is_train:\n        ann_path = f'{base_path}train_annotation/{lab}/{vid}.parquet'\n        anns = pd.read_parquet(ann_path)\n        def expand_to_frames(anns, total_frames):\n            frame_labels = np.full(total_frames, 'background')\n            for _, r in anns.iterrows():\n                action = f\"{int(r['agent_id'])}_{int(r['target_id'])}_{r['action']}\"\n                for f in range(int(r['start_frame']), min(int(r['stop_frame']) + 1, total_frames)):\n                    frame_labels[f] = action\n            return frame_labels\n        frame_labels = expand_to_frames(anns, total_frames)\n        track['label'] = track['video_frame'].apply(\n            lambda f: frame_labels[int(f)] if int(f) < len(frame_labels) else 'background'\n        )\n    # Per-mouse features\n    bc = track[track['bodypart'] == 'body_center'].sort_values(['mouse_id', 'video_frame'])\n    bc['dx'] = bc.groupby('mouse_id')['x_cm'].diff()\n    bc['dy'] = bc.groupby('mouse_id')['y_cm'].diff()\n    bc['speed'] = np.sqrt(bc['dx']**2 + bc['dy']**2)\n    bc['heading'] = np.arctan2(bc['dy'], bc['dx'])\n    bc['dist_center'] = np.sqrt(bc['x_cm']**2 + bc['y_cm']**2)\n    # Pairwise nose distances\n    nose = track[track['bodypart'] == 'nose']\n    pairs = [(1,2), (1,3), (1,4), (2,3), (2,4), (3,4)]\n    pair_dists = defaultdict(list)\n    pair_dists['video_frame'] = []\n    for frame in nose['video_frame'].unique():\n        frame_nose = nose[nose['video_frame'] == frame]\n        pair_dists['video_frame'].append(frame)\n        for a, b in pairs:\n            pos_a = frame_nose[frame_nose['mouse_id'] == a][['x_cm', 'y_cm']].mean()\n            pos_b = frame_nose[frame_nose['mouse_id'] == b][['x_cm', 'y_cm']].mean()\n            dist = np.sqrt((pos_a['x_cm'] - pos_b['x_cm'])**2 + (pos_a['y_cm'] - pos_b['y_cm'])**2) if not pos_a.empty and not pos_b.empty else 0\n            pair_dists[f'dist_{a}_{b}'].append(dist)\n    pair_df = pd.DataFrame(pair_dists)\n    # Merge\n    features_df = bc.merge(pair_df, on='video_frame', how='left').fillna(0)\n    if is_train:\n        features_df['label'] = features_df['video_frame'].apply(\n            lambda f: frame_labels[int(f)] if int(f) < len(frame_labels) else 'background'\n        )\n    return features_df, anns\n# Training\nnon_mabe_labs = ~train_meta['lab_id'].str.contains('MABe22')\ntrain_non_mabe = train_meta[non_mabe_labs]\nall_behaviors = []\nfor behaviors in train_non_mabe['behaviors_labeled']:\n    all_behaviors.extend(behaviors)\naction_types = [parse_behavior_label(b)[2] for b in all_behaviors if parse_behavior_label(b)]\nglobal_actions = sorted(set(action_types))\nprint(f\"Global actions from non-MABe22: {global_actions}\")\nprint(f\"Number of unique classes: {len(global_actions)}\")\nprint(\"Filtering non-MABe22 train data...\")\nannotated_non_mabe = train_non_mabe[train_non_mabe['behaviors_labeled'].apply(len) > 0]\nprint(f\"Non-MABe22 annotated videos: {len(annotated_non_mabe)}\")\ndef log_memory(phase):\n    ram_percent = psutil.virtual_memory().percent\n    print(f\"{phase} - RAM usage: {ram_percent:.2f}%\")\nlog_memory(\"After metadata load\")\nunique_labels = set(['background'])\ndef process_video(i):\n    row = annotated_non_mabe.iloc[i]\n    lab, vid = row['lab_id'], row['video_id']\n    if lab == 'PleasantMeerkat' and vid == '1375833299':\n        return None, set()\n    df, _ = load_video_features(lab, vid)\n    print(f\"Loaded features for {lab}/{vid}\")\n    label_set = set(df['label'].unique()) if 'label' in df.columns else set()\n    return df, label_set\nprint(\"Loading features in parallel...\")\nresults = Parallel(n_jobs=-1)(delayed(process_video)(i) for i in range(len(annotated_non_mabe)))\nall_dfs = [r[0] for r in results if r[0] is not None]\nfor r in results:\n    unique_labels.update(r[1])\nlog_memory(\"After feature loading\")\nglobal_le = LabelEncoder()\nglobal_le.fit(sorted(unique_labels))\nprint(f\"Global LE fitted on {len(global_le.classes_)} unique labels: {global_le.classes_}\")\nall_features = []\nfor i, df in enumerate(all_dfs):\n    if 'label' in df.columns and len(df[df['label'] != 'background']) > 0:\n        df['video_id'] = annotated_non_mabe.iloc[i]['video_id']\n        all_features.append(df)\nprint(f\"Loaded {len(all_features)} videos with non-background labels.\")\ndel all_dfs\ngc.collect()\nlog_memory(\"After label encoding\")\nif all_features:\n    feature_cols = ['x_cm', 'y_cm', 'speed', 'heading', 'dist_center', 'dist_1_2', 'dist_1_3', 'dist_1_4', 'dist_2_3', 'dist_2_4', 'dist_3_4', 'mouse_id']\n    mouse_categories = [1, 2, 3, 4]\n    X_list = [df[feature_cols].astype(np.float32) for df in all_features]\n    X_all = pd.concat(X_list)\n    X_all['mouse_id'] = pd.Categorical(X_all['mouse_id'], categories=mouse_categories)\n    X_all = pd.get_dummies(X_all, columns=['mouse_id'], prefix='mouse').fillna(0).astype(np.float32)\n    full_labels = [df['label'].tolist() for df in all_features]\n    full_labels = [lbl for sublist in full_labels for lbl in sublist]\n    y_all = np.array([global_le.transform([lbl])[0] for lbl in full_labels], dtype=np.int32)\n    print(f\"Total training samples: {len(y_all)}\")\n    print(\"Training RF on full data...\")\n    rf = RandomForestClassifier(n_estimators=200, random_state=42, class_weight='balanced', n_jobs=-1, max_depth=10, max_features=0.8)\n    rf.fit(X_all, y_all)\n    rf_train_acc = accuracy_score(y_all, rf.predict(X_all))\n    print(f\"RF Train accuracy: {rf_train_acc:.4f}\")\n    feat_cols = X_all.columns.tolist()\n    del X_all, y_all\n    gc.collect()\n    print(\"RF ready.\")\n    log_memory(\"After RF training\")\n# =============================================================================\n# INFERENCE ON TEST SET & GENERATE SUBMISSION\n# =============================================================================\nsubmission_path = '/kaggle/working/submission.csv'\nsubmission_rows = []\nrow_id = 0\nprocessed_videos = set()\nif os.path.exists(submission_path):\n    existing_sub = pd.read_csv(submission_path)\n    submission_rows = existing_sub.to_dict('records')\n    row_id = existing_sub['row_id'].max() + 1 if not existing_sub.empty else 0\n    processed_videos = set(existing_sub['video_id'].unique())\n    print(f\"Loaded existing submission with {len(submission_rows)} rows. Resuming from row_id {row_id}. Processed videos: {len(processed_videos)}\")\nfeat_cols = feat_cols if 'feat_cols' in locals() else []\nprint(\"Starting inference on test set...\")\nmouse_categories = [1, 2, 3, 4]\nfor i, (idx, test_row) in enumerate(test_meta.iterrows()):\n    lab, vid = test_row['lab_id'], test_row['video_id']\n    if vid in processed_videos:\n        print(f\"Skipping already processed test video {i+1}/{len(test_meta)}: {lab}/{vid}\")\n        continue\n    print(f\"Processing test video: {lab}/{vid}\")\n    df_test, _ = load_video_features(lab, vid, is_train=False)\n    print(f\"Loaded test features for {lab}/{vid}\")\n    df_test['video_id'] = vid\n    X_test = df_test[['x_cm', 'y_cm', 'speed', 'heading', 'dist_center', 'dist_1_2', 'dist_1_3', 'dist_1_4', 'dist_2_3', 'dist_2_4', 'dist_3_4', 'mouse_id']].fillna(0).astype(np.float32)\n    X_test['mouse_id'] = pd.Categorical(X_test['mouse_id'], categories=mouse_categories)\n    X_test = pd.get_dummies(X_test, columns=['mouse_id'], prefix='mouse').fillna(0).astype(np.float32)\n    if len(feat_cols) > 0:\n        missing_cols = set(feat_cols) - set(X_test.columns)\n        for col in missing_cols:\n            X_test[col] = 0\n        X_test = X_test[feat_cols].astype(np.float32)\n    rf_probs = rf.predict_proba(X_test)\n    frame_probs = rf_probs\n    frame_preds = np.argmax(frame_probs, axis=1)\n    frame_max_probs = np.max(frame_probs, axis=1)\n    def test_preds_to_intervals(preds, probs, frames, video_behaviors, thresh=0.3):\n        intervals = []\n        global row_id\n        current_action_idx = None\n        start = None\n        frames = frames.reset_index(drop=True)  # Ensure index starts from 0\n        for i, (pred, prob, frame) in enumerate(zip(preds, probs, frames)):\n            if prob > thresh and pred != current_action_idx:\n                if current_action_idx is not None:\n                    action_str = global_le.inverse_transform([current_action_idx])[0]\n                    parts = action_str.split('_')\n                    if len(parts) == 3:\n                        agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                        start_frame = int(start * 10)\n                        stop_frame = int(frame * 10 - 1)\n                        if stop_frame >= start_frame and any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                            intervals.append({\n                                'row_id': row_id,\n                                'video_id': int(vid),  # Ensure integer type\n                                'agent_id': f'mouse{agent}',\n                                'target_id': f'mouse{target}',\n                                'action': action,\n                                'start_frame': start_frame,\n                                'stop_frame': stop_frame\n                            })\n                            row_id += 1\n                current_action_idx = pred\n                start = frame\n            elif prob <= thresh and current_action_idx is not None:\n                action_str = global_le.inverse_transform([current_action_idx])[0]\n                parts = action_str.split('_')\n                if len(parts) == 3:\n                    agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                    start_frame = int(start * 10)\n                    stop_frame = int(frame * 10 - 1)\n                    if stop_frame >= start_frame and any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                        intervals.append({\n                            'row_id': row_id,\n                            'video_id': int(vid),  # Ensure integer type\n                            'agent_id': f'mouse{agent}',\n                            'target_id': f'mouse{target}',\n                            'action': action,\n                            'start_frame': start_frame,\n                            'stop_frame': stop_frame\n                        })\n                        row_id += 1\n                current_action_idx = None\n        if current_action_idx is not None:\n            action_str = global_le.inverse_transform([current_action_idx])[0]\n            parts = action_str.split('_')\n            if len(parts) == 3:\n                agent, target, action = int(parts[0]), int(parts[1]), parts[2]\n                start_frame = int(start * 10)\n                stop_frame = int(frames.iloc[-1] * 10 - 1)\n                if stop_frame >= start_frame and any(parse_behavior_label(b) == (agent, target, action) for b in video_behaviors):\n                    intervals.append({\n                        'row_id': row_id,\n                        'video_id': int(vid),  # Ensure integer type\n                        'agent_id': f'mouse{agent}',\n                        'target_id': f'mouse{target}',\n                        'action': action,\n                        'start_frame': start_frame,\n                        'stop_frame': stop_frame\n                    })\n                    row_id += 1\n        return intervals\n    video_behaviors = test_row['behaviors_labeled']\n    pred_ints = test_preds_to_intervals(frame_preds, frame_max_probs, df_test['video_frame'], video_behaviors)\n    submission_rows.extend(pred_ints)\n    print(f\"Detected {len(pred_ints)} intervals for {vid}\")\n    del df_test, X_test\n    gc.collect()\n# Final submission formatting to match competition requirements\nsubmission_df = pd.DataFrame(submission_rows)\nif len(submission_df) > 0:\n    submission_df = submission_df[['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']]\n    submission_df['video_id'] = submission_df['video_id'].astype(int)      # Ensure integer type\n    submission_df['row_id'] = submission_df['row_id'].astype(int)          # Ensure integer type\n    submission_df['start_frame'] = submission_df['start_frame'].astype(int) # Ensure integer type\n    submission_df['stop_frame'] = submission_df['stop_frame'].astype(int)   # Ensure integer type\n    submission_df.sort_values(['video_id', 'start_frame'], inplace=True)\n    submission_df['row_id'] = range(len(submission_df))  # Reset row_id to be sequential\nelse:\n    # Create empty submission with correct format if no predictions\n    submission_df = pd.DataFrame(columns=['row_id', 'video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame'])\nsubmission_df.to_csv(submission_path, index=False)\nprint(f\"Submission saved to {submission_path}\")\nprint(f\"Final submission shape: {submission_df.shape}\")\nprint(\"Submission columns and dtypes:\")\nprint(submission_df.dtypes)\nprint(submission_df.head())","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null}]}