{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Final version for commit\n\n\"\"\"\nNOTEBOOK A: PREPROCESSOR\n- Processes all 8,789 videos with feature engineering\n- Saves processed data for each video to separate files\n- No training, just data preparation\n- Runtime: ~4.5 hours on CPU\n\"\"\"\n\nimport gc\nfrom tqdm import tqdm\nimport pandas as pd\nimport numpy as np\nimport os\nfrom itertools import permutations\nfrom collections import defaultdict\nimport warnings\nfrom scipy.signal import welch\n\n# Feature engineering functions\ndef add_olfactory_intent_features(df_full_flat, agent_id, target_id, fps):\n    features = {}\n    try:\n        v_ba_x = df_full_flat[f'x_{agent_id}_nose'] - df_full_flat[f'x_{agent_id}_tail_base']\n        v_ba_y = df_full_flat[f'y_{agent_id}_nose'] - df_full_flat[f'y_{agent_id}_tail_base']\n        v_t_x = df_full_flat[f'x_{target_id}_tail_base'] - df_full_flat[f'x_{agent_id}_nose']\n        v_t_y = df_full_flat[f'y_{target_id}_tail_base'] - df_full_flat[f'y_{agent_id}_nose']\n        dot_product = v_ba_x * v_t_x + v_ba_y * v_t_y\n        norm_ba = np.sqrt(v_ba_x**2 + v_ba_y**2)\n        norm_t = np.sqrt(v_t_x**2 + v_t_y**2)\n        cosine_angle = np.clip(dot_product / (norm_ba * norm_t + 1e-6), -1.0, 1.0)\n        features['hae_genital_deg'] = np.degrees(np.arccos(cosine_angle))\n        \n        nose_speed = np.sqrt(df_full_flat[f'x_{agent_id}_nose_vel']**2 + df_full_flat[f'y_{agent_id}_nose_vel']**2)\n        def calculate_psd_optimized(x):\n            if len(x) < 8: return 0\n            try:\n                nperseg = min(len(x), max(8, len(x)//4))\n                freqs, psd = welch(x, fs=fps, nperseg=nperseg, noverlap=nperseg//2)\n                band_mask = (freqs >= 8.0) & (freqs <= 15.0)\n                return np.sum(psd[band_mask]) if np.any(band_mask) else 0\n            except: return 0\n        \n        window_size = max(8, int(fps / 4))\n        features['p_sniff_agent'] = nose_speed.rolling(window=window_size, center=True, min_periods=8).apply(calculate_psd_optimized, raw=True).fillna(0)\n    except KeyError: pass\n    return pd.DataFrame(features, index=df_full_flat.index).fillna(0)\n\ndef add_lunge_features(df_full_flat, agent_id, fps):\n    features = {}\n    try:\n        body_length = np.sqrt((df_full_flat[f'x_{agent_id}_nose'] - df_full_flat[f'x_{agent_id}_tail_base'])**2 + (df_full_flat[f'y_{agent_id}_nose'] - df_full_flat[f'y_{agent_id}_tail_base'])**2)\n        reference_length = body_length.rolling(window=int(fps*5), min_periods=1, center=True).mean()\n        features['dbcr_agent'] = body_length / (reference_length + 1e-6)\n        \n        center_bp = 'body_center' if f'x_{agent_id}_body_center' in df_full_flat.columns else 'neck'\n        accel_x = df_full_flat[f'x_{agent_id}_{center_bp}_vel'].diff() * fps\n        accel_y = df_full_flat[f'y_{agent_id}_{center_bp}_vel'].diff() * fps\n        features['acom_agent'] = np.sqrt(accel_x**2 + accel_y**2)\n        \n        is_compressed = (features['dbcr_agent'].shift(int(fps * 0.05)) < 0.95)\n        is_accelerating = (features['acom_agent'] > features['acom_agent'].mean() + 3 * features['acom_agent'].std())\n        features['lunge_signature'] = (is_compressed & is_accelerating).astype(int)\n    except KeyError: pass\n    return pd.DataFrame(features, index=df_full_flat.index).fillna(0)\n\ndef add_withdrawal_features(df_full_flat, agent_id, fps):\n    features = {}\n    try:\n        center_bp = 'body_center' if f'x_{agent_id}_body_center' in df_full_flat.columns else 'neck'\n        vel_x = df_full_flat[f'x_{agent_id}_{center_bp}'].diff() * fps\n        accel_x = vel_x.diff() * fps\n        jerk_x = accel_x.diff() * fps\n        vel_y = df_full_flat[f'y_{agent_id}_{center_bp}'].diff() * fps\n        accel_y = vel_y.diff() * fps\n        jerk_y = accel_y.diff() * fps\n        features['jerk_magnitude_agent'] = np.sqrt(jerk_x**2 + jerk_y**2)\n        \n        if f'x_{agent_id}_hip_left' in df_full_flat.columns and f'x_{agent_id}_hip_right' in df_full_flat.columns:\n            hip_width = np.sqrt((df_full_flat[f'x_{agent_id}_hip_left'] - df_full_flat[f'x_{agent_id}_hip_right'])**2 + (df_full_flat[f'y_{agent_id}_hip_left'] - df_full_flat[f'y_{agent_id}_hip_right'])**2)\n            features['hip_width_agent'] = hip_width\n        \n        speed = np.sqrt(vel_x**2 + vel_y**2)\n        is_stationary = speed < 2.0\n        features['is_submitting_agent'] = is_stationary.rolling(window=int(fps)).apply(np.all, raw=True)\n    except KeyError: pass\n    return pd.DataFrame(features, index=df_full_flat.index).fillna(0)\n\ndef add_advanced_kinematic_features(df_flat, mouse_ids, pix_per_cm):\n    for mid in mouse_ids:\n        nose_x, nose_y = f'x_{mid}_nose', f'y_{mid}_nose'\n        tail_x, tail_y = f'x_{mid}_tail_base', f'y_{mid}_tail_base'\n        if nose_x in df_flat.columns and tail_x in df_flat.columns:\n            body_length_px = np.sqrt((df_flat[nose_x] - df_flat[tail_x])**2 + (df_flat[nose_y] - df_flat[tail_y])**2)\n            df_flat[f'body_length_cm_{mid}'] = body_length_px / pix_per_cm\n            df_flat[f'body_length_vel_{mid}'] = df_flat[f'body_length_cm_{mid}'].diff()\n            body_length_accel = df_flat[f'body_length_vel_{mid}'].diff()\n            df_flat[f'body_flinch_signal_{mid}'] = body_length_accel.fillna(0)\n        \n        center_bp = 'body_center' if f'x_{mid}_body_center' in df_flat.columns else 'neck'\n        center_bp_vel_x = f'x_{mid}_{center_bp}_vel'\n        if center_bp_vel_x in df_flat.columns:\n            accel_x = df_flat[f'x_{mid}_{center_bp}_vel'].diff()\n            accel_y = df_flat[f'y_{mid}_{center_bp}_vel'].diff()\n            window = 5\n            accel_x_var = accel_x.rolling(window, center=True, min_periods=1).var()\n            accel_y_var = accel_y.rolling(window, center=True, min_periods=1).var()\n            df_flat[f'jitter_metric_{mid}'] = np.sqrt(accel_x_var**2 + accel_y_var**2).fillna(0)\n    return df_flat\n\ndef add_sniff_detection_features(df_full_flat, mouse_ids, fps):\n    for mid in mouse_ids:\n        try:\n            nose_vel_x_col = f'x_{mid}_nose_vel'\n            nose_vel_y_col = f'y_{mid}_nose_vel'\n            if (nose_vel_x_col in df_full_flat.columns and nose_vel_y_col in df_full_flat.columns and not df_full_flat[nose_vel_x_col].isna().all()):\n                nose_speed = np.sqrt(df_full_flat[nose_vel_x_col]**2 + df_full_flat[nose_vel_y_col]**2)\n                window_size = min(int(fps), 10)\n                df_full_flat[f'sniff_intensity_{mid}'] = nose_speed.rolling(window=window_size, center=True, min_periods=1).var().fillna(0)\n            else:\n                df_full_flat[f'sniff_intensity_{mid}'] = 0\n        except (KeyError, TypeError, ValueError):\n            df_full_flat[f'sniff_intensity_{mid}'] = 0\n    return df_full_flat\n\ndef add_submission_specific_features(df_full_flat, mouse_ids, pix_per_cm, fps):\n    for mid in mouse_ids:\n        lat_L = f'x_{mid}_lateral_left'\n        lat_R = f'x_{mid}_lateral_right'\n        if lat_L in df_full_flat.columns and lat_R in df_full_flat.columns:\n            lateral_width_px = np.sqrt((df_full_flat[f'x_{mid}_lateral_left'] - df_full_flat[f'x_{mid}_lateral_right'])**2 + (df_full_flat[f'y_{mid}_lateral_left'] - df_full_flat[f'y_{mid}_lateral_right'])**2)\n            df_full_flat[f'lateral_width_cm_{mid}'] = lateral_width_px / pix_per_cm\n            \n            nose_x, tail_x = f'x_{mid}_nose', f'x_{mid}_tail_base'\n            if nose_x in df_full_flat.columns and tail_x in df_full_flat.columns:\n                longitudinal_length_px = np.sqrt((df_full_flat[nose_x] - df_full_flat[tail_x])**2 + (df_full_flat[f'y_{mid}_nose'] - df_full_flat[f'y_{mid}_tail_base'])**2)\n                df_full_flat[f'longitudinal_length_cm_{mid}'] = longitudinal_length_px / pix_per_cm\n                df_full_flat[f'compression_index_{mid}'] = df_full_flat[f'lateral_width_cm_{mid}'] / (df_full_flat[f'longitudinal_length_cm_{mid}'] + 1e-6)\n        \n        center_bp = 'body_center' if f'x_{mid}_body_center' in df_full_flat.columns else 'neck'\n        if f'x_{mid}_{center_bp}_vel' in df_full_flat.columns:\n            speed_cm_s = np.sqrt(df_full_flat[f'x_{mid}_{center_bp}_vel']**2 + df_full_flat[f'y_{mid}_{center_bp}_vel']**2) * fps / pix_per_cm\n            speed_3_frames_ago = speed_cm_s.shift(3)\n            threshold_high_speed = 10.0\n            threshold_low_speed = 1.0\n            df_full_flat[f'velocity_collapse_{mid}'] = ((speed_3_frames_ago > threshold_high_speed) & (speed_cm_s < threshold_low_speed)).astype(int)\n            \n            if f'x_{mid}_nose' in df_full_flat.columns and f'x_{mid}_neck' in df_full_flat.columns:\n                nose_neck_dist = np.sqrt((df_full_flat[f'x_{mid}_nose'] - df_full_flat[f'x_{mid}_neck'])**2 + (df_full_flat[f'y_{mid}_nose'] - df_full_flat[f'y_{mid}_neck'])**2)\n                nose_tail_dist = np.sqrt((df_full_flat[f'x_{mid}_nose'] - df_full_flat[f'x_{mid}_tail_base'])**2 + (df_full_flat[f'y_{mid}_nose'] - df_full_flat[f'y_{mid}_tail_base'])**2)\n                df_full_flat[f'head_tucking_ratio_{mid}'] = nose_neck_dist / (nose_tail_dist + 1e-6)\n    return df_full_flat\n\ndef create_pair_specific_features_robust(df_full_flat, agent_id, target_id, fps, pix_per_cm, parts_agent, parts_target):\n    features = {}\n    center_bp_agent = 'body_center' if 'body_center' in parts_agent else 'neck'\n    center_bp_target = 'body_center' if 'body_center' in parts_target else 'neck'\n    \n    try:\n        if (f'x_{agent_id}_{center_bp_agent}' in df_full_flat.columns and f'x_{target_id}_{center_bp_target}' in df_full_flat.columns):\n            dist_px = np.sqrt((df_full_flat[f'x_{agent_id}_{center_bp_agent}'] - df_full_flat[f'x_{target_id}_{center_bp_target}'])**2 + (df_full_flat[f'y_{agent_id}_{center_bp_agent}'] - df_full_flat[f'y_{target_id}_{center_bp_target}'])**2)\n            features['dist_center_cm_agent_target'] = dist_px / pix_per_cm\n    except (KeyError, TypeError): features['dist_center_cm_agent_target'] = 0\n    \n    try:\n        if f'x_{agent_id}_{center_bp_agent}_vel' in df_full_flat.columns:\n            agent_vel_x = df_full_flat[f'x_{agent_id}_{center_bp_agent}_vel']\n            agent_vel_y = df_full_flat[f'y_{agent_id}_{center_bp_agent}_vel']\n            features['speed_cm_s_agent'] = np.sqrt(agent_vel_x**2 + agent_vel_y**2) * fps / pix_per_cm\n    except (KeyError, TypeError): features['speed_cm_s_agent'] = 0\n    \n    try:\n        if f'x_{target_id}_{center_bp_target}_vel' in df_full_flat.columns:\n            target_vel_x = df_full_flat[f'x_{target_id}_{center_bp_target}_vel']\n            target_vel_y = df_full_flat[f'y_{target_id}_{center_bp_target}_vel']\n            features['speed_cm_s_target'] = np.sqrt(target_vel_x**2 + target_vel_y**2) * fps / pix_per_cm\n    except (KeyError, TypeError): features['speed_cm_s_target'] = 0\n    \n    if 'dist_center_cm_agent_target' in features:\n        try: features['relative_speed_cm_s_agent_target'] = features['dist_center_cm_agent_target'].diff() * fps\n        except (KeyError, TypeError): features['relative_speed_cm_s_agent_target'] = 0\n    \n    try:\n        if (f'x_{agent_id}_nose' in df_full_flat.columns and f'x_{agent_id}_tail_base' in df_full_flat.columns and f'x_{target_id}_{center_bp_target}' in df_full_flat.columns):\n            agent_angle = np.arctan2(df_full_flat[f'y_{agent_id}_nose'] - df_full_flat[f'y_{agent_id}_tail_base'], df_full_flat[f'x_{agent_id}_nose'] - df_full_flat[f'x_{agent_id}_tail_base'])\n            target_angle = np.arctan2(df_full_flat[f'y_{target_id}_{center_bp_target}'] - df_full_flat[f'y_{agent_id}_{center_bp_agent}'], df_full_flat[f'x_{target_id}_{center_bp_target}'] - df_full_flat[f'x_{agent_id}_{center_bp_agent}'])\n            relative_angle = target_angle - agent_angle\n            features['relative_angle_agent_target'] = (relative_angle + np.pi) % (2 * np.pi) - np.pi\n    except (KeyError, TypeError): features['relative_angle_agent_target'] = 0\n    \n    for mid, label in [(agent_id, 'agent'), (target_id, 'target')]:\n        try:\n            if f'body_length_cm_{mid}' in df_full_flat.columns:\n                features[f'body_length_cm_{label}'] = df_full_flat[f'body_length_cm_{mid}']\n            else: features[f'body_length_cm_{label}'] = 0\n        except (KeyError, TypeError): features[f'body_length_cm_{label}'] = 0\n        \n        try:\n            if f'sniff_intensity_{mid}' in df_full_flat.columns:\n                features[f'sniff_intensity_{label}'] = df_full_flat[f'sniff_intensity_{mid}']\n            else: features[f'sniff_intensity_{label}'] = 0\n        except (KeyError, TypeError): features[f'sniff_intensity_{label}'] = 0\n    \n    return pd.DataFrame(features, index=df_full_flat.index).fillna(0)\n\ndef engineer_features_for_video(video_id, tracking_path, metadata_df):\n    try:\n        video_metadata = metadata_df.loc[int(video_id)]\n        fps = video_metadata['frames_per_second']\n        pix_per_cm = video_metadata['pix_per_cm_approx']\n    except (KeyError, TypeError): fps, pix_per_cm = 30.0, 10.0\n    \n    try: df_long = pd.read_parquet(tracking_path)\n    except Exception: return None\n    \n    df_wide = df_long.pivot_table(index='video_frame', columns=['mouse_id', 'bodypart'], values=['x', 'y'])\n    df_wide_clean = df_wide.interpolate(method='linear', limit_direction='both', axis=0)\n    df_flat = df_wide_clean.copy()\n    df_flat.columns = ['_'.join(map(str, col)).replace(\"('\", \"\").replace(\"', '\", \"_\").replace(\"')\", \"\") for col in df_flat.columns]\n    \n    mouse_ids = sorted(df_long['mouse_id'].unique())\n    \n    try: df_flat = add_advanced_kinematic_features(df_flat, mouse_ids, pix_per_cm)\n    except Exception: return None\n    \n    try:\n        velocity_df = df_flat.diff()\n        velocity_df.columns = [f'{col}_vel' for col in df_flat.columns]\n        df_full_flat = pd.concat([df_flat, velocity_df], axis=1)\n    except Exception: return None\n    \n    try: df_full_flat = add_sniff_detection_features(df_full_flat, mouse_ids, fps)\n    except Exception: return None\n    \n    try: df_full_flat = add_submission_specific_features(df_full_flat, mouse_ids, pix_per_cm, fps)\n    except Exception: return None\n    \n    try:\n        all_pair_features_list = []\n        for agent_id, target_id in permutations(mouse_ids, 2):\n            try:\n                parts_agent = set(df_long[df_long['mouse_id'] == agent_id]['bodypart'])\n                parts_target = set(df_long[df_long['mouse_id'] == target_id]['bodypart'])\n                \n                pair_features = create_pair_specific_features_robust(df_full_flat, agent_id, target_id, fps, pix_per_cm, parts_agent, parts_target)\n                \n                # Add rolling averages\n                for window in [5, 15]:\n                    for col in ['dist_center_cm_agent_target', 'relative_speed_cm_s_agent_target']:\n                        if col in pair_features.columns and not pair_features[col].isna().all():\n                            try: pair_features[f'{col}_mean_{window}f'] = pair_features[col].rolling(window=window, center=True, min_periods=1).mean()\n                            except (KeyError, TypeError): pair_features[f'{col}_mean_{window}f'] = 0\n                \n                # Add advanced features\n                try:\n                    olfactory_features = add_olfactory_intent_features(df_full_flat, agent_id, target_id, fps)\n                    lunge_features = add_lunge_features(df_full_flat, agent_id, fps)\n                    withdrawal_features = add_withdrawal_features(df_full_flat, agent_id, fps)\n                    \n                    for df_feat in [olfactory_features, lunge_features, withdrawal_features]:\n                        if not df_feat.empty:\n                            for col in df_feat.columns:\n                                pair_features[col] = df_feat[col]\n                except Exception: pass\n                \n                pair_features['agent_id'] = agent_id\n                pair_features['target_id'] = target_id\n                pair_features['video_frame'] = pair_features.index\n                all_pair_features_list.append(pair_features)\n            except Exception: continue\n        \n        if not all_pair_features_list: return None\n        \n        final_video_features = pd.concat(all_pair_features_list, ignore_index=True)\n        return final_video_features\n    except Exception: return None\n\ndef create_training_examples(full_features_df, annotation_df, video_id, mouse_ids, metadata_df):\n    try:\n        video_metadata = metadata_df.loc[int(video_id)]\n    except (KeyError, TypeError):\n        video_metadata = {}\n    \n    mouse_info = {}\n    for i in range(1, 5):\n        if f'mouse{i}_strain' in video_metadata and pd.notna(video_metadata.get(f'mouse{i}_strain')):\n            mouse_info[i] = {\n                'strain': video_metadata.get(f'mouse{i}_strain', 'unknown'),\n                'sex': video_metadata.get(f'mouse{i}_sex', 'unknown'),\n            }\n    \n    X_pieces, y_pieces, groups_pieces = [], [], []\n    \n    for _, event in annotation_df.iterrows():\n        agent, target, action = int(event['agent_id']), int(event['target_id']), event['action']\n        start, end = int(event['start_frame']), int(event['stop_frame'])\n        \n        pair_mask = (full_features_df['agent_id'] == agent) & (full_features_df['target_id'] == target)\n        frame_mask = (full_features_df['video_frame'] >= start) & (full_features_df['video_frame'] <= end)\n        event_data = full_features_df[pair_mask & frame_mask].copy()\n        \n        if event_data.empty: continue\n        \n        event_data['lab_id'] = video_metadata.get('lab_id', 'unknown')\n        event_data['agent_strain'] = mouse_info.get(agent, {}).get('strain', 'unknown')\n        event_data['target_strain'] = mouse_info.get(target, {}).get('strain', 'unknown')\n        event_data['agent_sex'] = mouse_info.get(agent, {}).get('sex', 'unknown')\n        event_data['target_sex'] = mouse_info.get(target, {}).get('sex', 'unknown')\n        event_data['arena_type'] = video_metadata.get('arena_type', 'unknown')\n        event_data['tracking_method'] = video_metadata.get('tracking_method', 'unknown')\n        event_data['lights_on'] = video_metadata.get('lights_on', 'unknown')\n        \n        X_pieces.append(event_data)\n        y_pieces.extend([action] * len(event_data))\n        groups_pieces.extend([video_id] * len(event_data))\n    \n    if not X_pieces: return None\n    \n    positive_frames = set()\n    for _, event in annotation_df.iterrows():\n        positive_frames.update(range(event['start_frame'], event['stop_frame'] + 1))\n    \n    all_frames = set(full_features_df['video_frame'].unique())\n    background_frames = list(all_frames - positive_frames)\n    \n    num_pos_samples = sum(len(p) for p in X_pieces)\n    num_neg_to_sample = min(num_pos_samples // 2, len(background_frames))\n    \n    if num_neg_to_sample > 0:\n        sampled_frames = np.random.choice(background_frames, size=num_neg_to_sample, replace=False)\n        \n        for frame in sampled_frames:\n            agent, target = np.random.choice(mouse_ids, 2, replace=False)\n            \n            pair_mask = (full_features_df['agent_id'] == agent) & (full_features_df['target_id'] == target)\n            frame_mask = (full_features_df['video_frame'] == frame)\n            neg_data = full_features_df[pair_mask & frame_mask].copy()\n            \n            if neg_data.empty: continue\n            \n            neg_data['lab_id'] = video_metadata.get('lab_id', 'unknown')\n            neg_data['agent_strain'] = mouse_info.get(agent, {}).get('strain', 'unknown')\n            neg_data['target_strain'] = mouse_info.get(target, {}).get('strain', 'unknown')\n            neg_data['agent_sex'] = mouse_info.get(agent, {}).get('sex', 'unknown')\n            neg_data['target_sex'] = mouse_info.get(target, {}).get('sex', 'unknown')\n            neg_data['arena_type'] = video_metadata.get('arena_type', 'unknown')\n            neg_data['tracking_method'] = video_metadata.get('tracking_method', 'unknown')\n            neg_data['lights_on'] = video_metadata.get('lights_on', 'unknown')\n            \n            X_pieces.append(neg_data)\n            y_pieces.extend(['background'] * len(neg_data))\n            groups_pieces.extend([video_id] * len(neg_data))\n    \n    X_final = pd.concat(X_pieces, ignore_index=True)\n    y_final = pd.Series(y_pieces)\n    groups_final = pd.Series(groups_pieces)\n    \n    return X_final, y_final, groups_final\n\ndef check_files_exist(video_id, data_dir):\n    tracking_base_dir = os.path.join(data_dir, \"train_tracking\")\n    annotation_base_dir = os.path.join(data_dir, \"train_annotation\")\n    all_subsets = [d for d in os.listdir(tracking_base_dir) if os.path.isdir(os.path.join(tracking_base_dir, d))]\n    for subset in all_subsets:\n        tracking_path = os.path.join(tracking_base_dir, subset, f\"{video_id}.parquet\")\n        annotation_path = os.path.join(annotation_base_dir, subset, f\"{video_id}.parquet\")\n        if os.path.exists(tracking_path) and os.path.exists(annotation_path):\n            return tracking_path, annotation_path\n    return None, None\n\n# Configuration\nCONFIG = {'DATA_DIR': \"/kaggle/input/MABe-mouse-behavior-detection/\"}\nDATA_DIR = CONFIG['DATA_DIR']\n\nprint(\"🚀 Starting Stage 1A: Data Preprocessing\")\nmetadata_df = pd.read_csv(os.path.join(DATA_DIR, \"train.csv\")).set_index(\"video_id\")\nprint(\"Metadata loaded successfully.\")\n\ntracking_base_dir = os.path.join(DATA_DIR, \"train_tracking\")\nall_subsets = [d for d in os.listdir(tracking_base_dir) if os.path.isdir(os.path.join(tracking_base_dir, d))]\nVIDEO_IDS_TO_PROCESS = []\nfor subset in all_subsets:\n    tracking_dir = os.path.join(tracking_base_dir, subset)\n    subset_videos = [f.split(\".\")[0] for f in os.listdir(tracking_dir) if f.endswith('.parquet')]\n    VIDEO_IDS_TO_PROCESS.extend(subset_videos)\n\nprint(f\"Found {len(VIDEO_IDS_TO_PROCESS)} videos to process.\")\n\n# Main preprocessing work\nprint(\"\\n=== Building and Saving Training Examples Video by Video ===\")\nos.makedirs('/kaggle/working/processed_videos', exist_ok=True)\nprocessed_count = 0\n\nfor i, video_id in enumerate(tqdm(VIDEO_IDS_TO_PROCESS, desc=\"Processing Training Videos\")):\n    try:\n        tracking_path, annotation_path = check_files_exist(video_id, DATA_DIR)\n        if tracking_path is None: continue\n\n        full_video_features = engineer_features_for_video(video_id, tracking_path, metadata_df)\n        if full_video_features is None: continue\n\n        annotation_df = pd.read_parquet(annotation_path)\n        df_long = pd.read_parquet(tracking_path)\n        mouse_ids = sorted(df_long['mouse_id'].unique())\n        result = create_training_examples(full_video_features, annotation_df, video_id, mouse_ids, metadata_df)\n\n        if result:\n            X_video, y_video, groups_video = result\n            temp_df = X_video.copy()\n            temp_df['labels'] = y_video\n            temp_df['groups'] = groups_video\n            temp_df.to_parquet(f'/kaggle/working/processed_videos/video_{video_id}.parquet')\n            processed_count += 1\n            \n            # Clean up memory\n            del X_video, y_video, groups_video, temp_df, full_video_features, annotation_df, df_long\n            if i % 50 == 0: gc.collect()\n\n    except Exception as e:\n        continue\n\nprint(f\"\\n✅ Preprocessing Complete! Successfully processed and saved {processed_count} videos.\")\nprint(\"The output of this notebook is the '/kaggle/working/processed_videos' directory.\")\nprint(\"You can now commit this notebook and use its output as input for your training notebook.\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}