{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13658340,"sourceType":"datasetVersion","datasetId":8683601}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\ndirectory = '/kaggle/input/MABe-mouse-behavior-detection/'\npd.set_option('display.max_columns',100)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\n\n# ============================================================\n# 📦 Imports\n# ============================================================\nimport pandas as pd\nimport numpy as np\nimport re, ast, os, gc, warnings\nfrom tqdm import tqdm\n\nwarnings.filterwarnings(\"ignore\")\nnp.seterr(invalid=\"ignore\")\n\n# ============================================================\n# 1️⃣ Parse light/time info per mouse\n# ============================================================\ndef parse_light_info(df, col, num):\n    pattern = r\"day\\s*(\\d+)\\s*-\\s*(\\d{1,2}):(\\d{2})\\s*\\(([^)]+)\\)\"\n    base_col = col.removeprefix(f\"mouse{num}_\")\n    time_col = f\"mouse{num}_{base_col}_time_in_hours\"\n    light_col = f\"mouse{num}_{base_col}_light_status\"\n\n    def extract(entry):\n        if isinstance(entry, str):\n            m = re.match(pattern, entry.lower().strip())\n            if m:\n                day, hour, minute, status = m.groups()\n                return [int(day)*24 + int(hour) + int(minute)/60, status]\n        return [-1, \"unknown\"]\n\n    vals = df[col].map(extract)\n    df[time_col] = [v[0] for v in vals]\n    df[light_col] = [v[1] for v in vals]\n    return df\n\n\n# ============================================================\n# 2️⃣ Behavior parsing\n# ============================================================\ndef safe_parse_behavior_set(x):\n    if pd.isna(x) or not isinstance(x, str) or not x.strip():\n        return set()\n    try:\n        parsed = ast.literal_eval(x)\n        if not isinstance(parsed, (list, set)):\n            return set()\n        result = set()\n        for i in parsed:\n            if isinstance(i, str):\n                parts = i.split(',')\n                if len(parts) == 3:\n                    if parts[1].strip() == \"self\":\n                        result.add(f\"self, {parts[2].strip()}\")\n                    else:\n                        result.add(parts[2].strip())\n        return result\n    except Exception:\n        return set()\n\n\ndef prepare_behavior_sets(train):\n    print(\"→ Parsing behavior sets...\")\n    tqdm.pandas()\n    train[\"behavior_set\"] = train[\"behaviors_labeled\"].progress_apply(safe_parse_behavior_set)\n    final_set = set().union(*train[\"behavior_set\"])\n    print(f\"   ✅ Unique behaviors: {len(final_set)}\")\n    train[\"behavior_set\"] = train[\"behavior_set\"].apply(lambda s: s if s else final_set)\n    return train, final_set\n\n\n# ============================================================\n# 3️⃣ Feature extraction (groupby + unstack = lean pivot)\n# ============================================================\ndef extract_mouse_features(df, body_parts, video_width_pix=None, video_height_pix=None, arena_width_cm=None, arena_height_cm=None):\n    # Group and unstack instead of pivot_table (much leaner)\n    df = (\n        df.groupby([\"video_frame\", \"mouse_id\", \"bodypart\"], observed=True)[[\"x\", \"y\"]]\n        .mean()\n        .unstack(\"bodypart\")\n    )\n\n    df.columns = [f\"{c1}_{c2}\" for c1, c2 in df.columns]\n    df = df.astype(\"float32\", copy=False).reset_index()\n\n    present_parts = [bp for bp in body_parts if f\"x_{bp}\" in df.columns]\n    if not present_parts:\n        return pd.DataFrame()\n\n    arr_x = df[[f\"x_{bp}\" for bp in present_parts]].to_numpy(dtype=np.float32, copy=False)\n    arr_y = df[[f\"y_{bp}\" for bp in present_parts]].to_numpy(dtype=np.float32, copy=False)\n\n    df[\"COM_x\"] = np.nanmean(arr_x, axis=1)\n    df[\"COM_y\"] = np.nanmean(arr_y, axis=1)\n\n    # ====================================================\n    # 🧮 Convert COM coordinates from pixels → centimeters\n    # ====================================================\n    if all(v is not None for v in [video_width_pix, arena_width_cm, video_height_pix, arena_height_cm]):\n        scale_x = video_width_pix / arena_width_cm\n        scale_y = video_height_pix / arena_height_cm\n        df[\"COM_x\"] = df[\"COM_x\"] / scale_x\n        df[\"COM_y\"] = df[\"COM_y\"] / scale_y\n    else:\n        print(\"⚠️ Missing scaling info; COM remains in pixels.\")\n\n\n    for bp in [\"ear_left\", \"ear_right\", \"tail_base\"]:\n        if f\"x_{bp}\" not in df.columns:\n            df[f\"x_{bp}\"] = np.nan\n            df[f\"y_{bp}\"] = np.nan\n\n    df[\"ear_mid_x\"] = df[[\"x_ear_left\", \"x_ear_right\"]].mean(axis=1)\n    df[\"ear_mid_y\"] = df[[\"y_ear_left\", \"y_ear_right\"]].mean(axis=1)\n\n    dx = df[\"x_tail_base\"] - df[\"ear_mid_x\"]\n    dy = df[\"y_tail_base\"] - df[\"ear_mid_y\"]\n    df[\"orientation_angle\"] = np.arctan2(dy, dx).astype(\"float32\")\n\n    df.sort_values([\"mouse_id\", \"video_frame\"], inplace=True, ignore_index=True)\n\n    df[[\"vel_x\", \"vel_y\"]] = (\n        df.groupby(\"mouse_id\", sort=False)[[\"COM_x\", \"COM_y\"]].diff().astype(\"float32\")\n    )\n    df[\"velocity\"] = np.hypot(df[\"vel_x\"], df[\"vel_y\"]).astype(\"float32\")\n    df[[\"accel_x\", \"accel_y\"]] = (\n        df.groupby(\"mouse_id\", sort=False)[[\"vel_x\", \"vel_y\"]].diff().astype(\"float32\")\n    )\n    df[\"acceleration\"] = np.hypot(df[\"accel_x\"], df[\"accel_y\"]).astype(\"float32\")\n\n    del arr_x, arr_y\n    gc.collect()\n\n    return df[\n        [\n            \"video_frame\",\n            \"mouse_id\",\n            \"COM_x\",\n            \"COM_y\",\n            \"orientation_angle\",\n            \"velocity\",\n            \"acceleration\",\n        ]\n    ].astype(\"float32\", copy=False)\n    \n\n# ============================================================\n# 4️⃣ Pairwise features\n# ============================================================\ndef compute_pairwise_features(df):\n    mice = df[\"mouse_id\"].unique()\n    out = []\n\n    for a in mice:\n        s = df[df.mouse_id == a].copy()\n        for b in mice:\n            if a == b:\n                continue\n\n            # Target mouse b\n            t = df[df.mouse_id == b][\n                [\"video_frame\", \"COM_x\", \"COM_y\", \"orientation_angle\", \"velocity\", \"acceleration\"]\n            ].rename(columns=lambda c: f\"{c}_tgt\" if c != \"video_frame\" else c)\n\n            merged = pd.merge(s, t, on=\"video_frame\", how=\"inner\", sort=False)\n\n            # Vector from mouse a → mouse b\n            dx = merged[\"COM_x_tgt\"] - merged[\"COM_x\"]\n            dy = merged[\"COM_y_tgt\"] - merged[\"COM_y\"]\n            dist = np.hypot(dx, dy).astype(\"float32\")\n            merged[\"distance\"] = dist\n\n            # Unit vector toward target\n            ux = dx / (dist + 1e-6)\n            uy = dy / (dist + 1e-6)\n\n            # Mouse a's velocity direction components\n            vx = merged.groupby(\"mouse_id\", sort=False)[\"COM_x\"].diff().fillna(0).astype(\"float32\")\n            vy = merged.groupby(\"mouse_id\", sort=False)[\"COM_y\"].diff().fillna(0).astype(\"float32\")\n\n            # Project velocity along the line toward target (dot product)\n            v_toward = vx * ux + vy * uy\n            merged[\"relative_velocity\"] = v_toward.astype(\"float32\")\n\n            # Same for acceleration\n            ax = vx.diff().fillna(0)\n            ay = vy.diff().fillna(0)\n            a_toward = ax * ux + ay * uy\n            merged[\"relative_acceleration\"] = a_toward.astype(\"float32\")\n\n            # Relative orientation: angle between mouse's facing direction and line to target\n            orientation_to_target = np.arctan2(dy, dx)\n            delta_angle = merged[\"orientation_angle\"] - orientation_to_target\n            merged[\"relative_orientation\"] = np.arctan2(np.sin(delta_angle), np.cos(delta_angle)).astype(\"float32\")\n\n            # Attach identifiers\n            merged[\"mouse_id_target\"] = b\n\n            # Keep both coordinates + kinematics\n            out.append(\n                merged[\n                    [\n                        \"video_frame\",\n                        \"mouse_id\",\n                        \"mouse_id_target\",\n                        \"COM_x\",\n                        \"COM_y\",\n                        \"COM_x_tgt\",\n                        \"COM_y_tgt\",\n                        \"distance\",\n                        \"relative_orientation\",\n                        \"relative_velocity\",\n                        \"relative_acceleration\",\n                    ]\n                ]\n            )\n\n    if not out:\n        return pd.DataFrame()\n    return pd.concat(out, ignore_index=True)\n\n\n\n# ============================================================\n# 5️⃣ Metadata enrichment\n# ============================================================\ndef enrich_with_metadata(df_pairs, train_row, num_mice=4):\n    if df_pairs.empty:\n        return df_pairs\n\n    meta_cols = [\n        \"lab_id\",\n        \"video_id\",\n        \"pix_per_cm_approx\",\n        \"video_width_pix\",\n        \"video_height_pix\",\n        \"arena_width_cm\",\n        \"arena_height_cm\",\n        \"arena_shape\",\n        \"arena_type\",\n        \"body_parts_tracked\",\n        \"behaviors_labeled\",\n        \"tracking_method\",\n        \"no_of_mice\",\n        \"behavior_set\",\n    ]\n    for c in meta_cols:\n        val = train_row.get(c, np.nan)\n        if isinstance(val, (list, set, tuple)):\n            val = \",\".join(map(str, val))\n        df_pairs[c] = val\n\n    mouse_fields = [\n        \"strain\",\n        \"sex\",\n        \"age\",\n        \"color\",\n        \"condition\",\n        \"condition_time_in_hours\",\n        \"condition_light_status\",\n    ]\n    for mid in range(1, num_mice + 1):\n        for f in mouse_fields:\n            v = train_row.get(f\"mouse{mid}_{f}\", np.nan)\n            mask_self = df_pairs[\"mouse_id\"] == mid\n            mask_tgt = df_pairs[\"mouse_id_target\"] == mid\n            if mask_self.any():\n                df_pairs.loc[mask_self, f\"mouse_{f}\"] = v\n            if mask_tgt.any():\n                df_pairs.loc[mask_tgt, f\"target_mouse_{f}\"] = v\n\n    return df_pairs\n    \n\n# ============================================================\n# 7️⃣ Annotation merger\n# ============================================================\ndef attach_annotations(df_pairs, lab_id, video_id, base_annot_path):\n    \"\"\"\n    Attach behavior annotations for matching agent-target pairs.\n    If annotation file not found, leaves 'Target' column blank.\n    \"\"\"\n    annot_path = f\"{base_annot_path}/{lab_id}/{video_id}.parquet\"\n\n    # If annotation missing, just create blank column\n    if not os.path.exists(annot_path):\n        tqdm.write(f\"⚠️ No annotation file for {lab_id}/{video_id}\")\n        df_pairs[\"Target\"] = np.nan\n        return df_pairs\n        \n    try:\n        ann = pd.read_parquet(annot_path)\n\n        # Expected columns: ['agent_id','target_id','action','start_frame','stop_frame']\n        if not set([\"agent_id\", \"target_id\", \"action\", \"start_frame\", \"stop_frame\"]).issubset(ann.columns):\n            tqdm.write(f\"⚠️ Annotation format invalid for {lab_id}/{video_id}\")\n            df_pairs[\"Target\"] = np.nan\n            return df_pairs\n\n        # Prepare annotation mapping per (agent,target)\n        df_pairs[\"Target\"] = np.nan  # initialize column\n\n        # Iterate over unique combinations for efficient assignment\n        for _, row in ann.iterrows():\n            agent = row[\"agent_id\"]\n            target = row[\"target_id\"]\n            start_f = int(row[\"start_frame\"])\n            stop_f = int(row[\"stop_frame\"])\n            act = str(row[\"action\"]).strip().lower()\n\n            # Mask: same agent, same target, frame within range\n            mask = (\n                (df_pairs[\"mouse_id\"] == agent)\n                & (df_pairs[\"mouse_id_target\"] == target)\n                & (df_pairs[\"video_frame\"].between(start_f, stop_f))\n            )\n\n            df_pairs.loc[mask, \"Target\"] = act\n\n        return df_pairs\n\n    except Exception as e:\n        tqdm.write(f\"❌ Error reading annotation for {lab_id}/{video_id}: {e}\")\n        df_pairs[\"Target\"] = np.nan\n        return df_pairs\n\n\n# ============================================================\n# 6️⃣ Full pipeline\n# ============================================================\ndef run_train_pipeline(train, body_parts, base_path, annot_base_path, chunk_save_interval=10):\n    print(\"→ Preparing metadata...\")\n    train, final_set = prepare_behavior_sets(train)\n    all_results = []\n    chunk_idx = 0\n\n    for _, row in tqdm(train.iterrows(), total=len(train), desc=\"Processing Labs/Videos\"):\n        df = df_features = df_pairs = None\n        try:\n            path = f\"{base_path}/{row['lab_id']}/{row['video_id']}.parquet\"\n            if not os.path.exists(path):\n                tqdm.write(f\"⚠️ Missing file: {path}\")\n                continue\n\n            # 1️⃣ Read tracking\n            df = pd.read_parquet(path, columns=[\"video_frame\", \"mouse_id\", \"bodypart\", \"x\", \"y\"])\n\n            # print(df.shape)\n            \n            # 2️⃣ Extract scaled features\n            df_features = extract_mouse_features(\n                df,\n                body_parts,\n                video_width_pix=row[\"video_width_pix\"],\n                video_height_pix=row[\"video_height_pix\"],\n                arena_width_cm=row[\"arena_width_cm\"],\n                arena_height_cm=row[\"arena_height_cm\"],\n            )\n\n            if df_features.empty:\n                continue\n\n            # 3️⃣ Pairwise interactions\n            df_pairs = compute_pairwise_features(df_features)\n\n            # 4️⃣ Add metadata\n            df_pairs = enrich_with_metadata(df_pairs, row)\n\n\n            # 5️⃣ Attach target annotations\n            # df_pairs = attach_annotations(\n            #     df_pairs,\n            #     lab_id=row[\"lab_id\"],\n            #     video_id=row[\"video_id\"],\n            #     base_annot_path=annot_base_path,\n            # )\n\n            drop_cols = [\n                \"pix_per_cm_approx\",\n                \"video_width_pix\",\n                \"video_height_pix\",\n                \"arena_width_cm\",\n                \"arena_height_cm\",\n                \"body_parts_tracked\",\n                \"behaviors_labeled\",\n            ]\n            df_pairs = df_pairs.drop(columns=[c for c in drop_cols if c in df_pairs.columns], errors=\"ignore\")\n\n            # Uncomment this to get the concatenated data set\n            all_results.append(df_pairs)\n\n            ## Uncomment this to store the data \n            # lab_dir = f\"/kaggle/working/Dataset/{row['lab_id']}\"\n            # os.makedirs(lab_dir, exist_ok=True)\n            \n            # save_path = f\"{lab_dir}/{row['video_id']}.parquet\"\n            # df_pairs.to_parquet(save_path, compression=\"snappy\")\n            \n            del df, df_features, df_pairs\n            gc.collect()\n\n        except Exception as e:\n            tqdm.write(f\"❌ Error processing {row.get('lab_id')} / {row.get('video_id')}: {e}\")\n        # finally:\n        #     del df, df_features, df_pairs\n        #     gc.collect()\n\n    # Final combine after loop\n    final_df = pd.concat(all_results, ignore_index=True) if all_results else pd.DataFrame()\n    print(f\"✅ Completed. Final shape: {final_df.shape}\")\n    return final_df, final_set\n\n\ntest = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/test.csv')\ntest.drop(columns=['mouse1_id', 'mouse2_id', 'mouse3_id', 'mouse4_id'], inplace=True)\n\nbody_parts = ['body_center', 'ear_left', 'ear_right', 'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft',\n              'headpiece_bottomfrontright', 'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright',\n              'lateral_left', 'lateral_right', 'neck', 'nose', 'tail_base', 'tail_midpoint', 'tail_tip', 'hip_left', 'hip_right', 'head', \n              'forepaw_left', 'forepaw_right', 'hindpaw_left', 'hindpaw_right', 'spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2']\n\nfinal_df, final_set = run_train_pipeline(\n    # Uncomment train & comment test_train for complete data (uncomment parquet file saving in run_train_pipeline as well)\n    test,\n    # train,\n    body_parts,\n    base_path=\"/kaggle/input/MABe-mouse-behavior-detection/test_tracking\",\n    annot_base_path=\"/kaggle/input/MABe-mouse-behavior-detection/train_annotation\"\n)\n\n\n\n# ============================================================\n# 📦 Imports\n# ============================================================\nimport pandas as pd\nimport numpy as np\nimport os, gc, warnings, joblib, json\nfrom tqdm import tqdm\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score, classification_report\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\nwarnings.filterwarnings(\"ignore\")\npd.options.mode.copy_on_write = True\n\n\n# ============================================================\n# 1️⃣ Data Loading\n# ============================================================\ndef load_behavior_data(df_meta, base_dir):\n    print(f\"🔹 Loading data from {base_dir} ...\")\n    dfs = []\n    total_rows = 0\n\n    for _, row in tqdm(df_meta.iterrows(), total=len(df_meta), desc=\"Loading videos\"):\n        lab_id, vid = row[\"lab_id\"], row[\"video_id\"]\n        fpath = f\"{base_dir}/{lab_id}/{vid}.parquet\"\n\n        if not os.path.exists(fpath):\n            print(f\"⚠️ Missing file: {fpath}\")\n            continue\n\n        try:\n            df = pd.read_parquet(fpath)\n            df[\"lab_id\"] = lab_id\n            df[\"video_id\"] = vid\n\n            for col in df.select_dtypes(\"float\").columns:\n                df[col] = pd.to_numeric(df[col], downcast=\"float\")\n            for col in df.select_dtypes(\"int\").columns:\n                df[col] = pd.to_numeric(df[col], downcast=\"integer\")\n\n            df = df[df['Target'].notna()]  # only labeled\n            # print(df.columns)\n            dfs.append(df)\n            total_rows += len(df)\n\n            if len(dfs) >= 10:  # memory control\n                dfs = [pd.concat(dfs, ignore_index=True)]\n                gc.collect()\n        except Exception as e:\n            print(f\"❌ Error loading {fpath}: {e}\")\n            continue\n\n    if not dfs:\n        raise ValueError(\"❌ No data files loaded!\")\n\n    full_df = pd.concat(dfs, ignore_index=True)\n    del dfs; gc.collect()\n    print(f\"✅ Loaded {len(full_df):,} rows total.\")\n    return full_df\n\n\n# ============================================================\n# 2️⃣ Feature Preparation\n# ============================================================\ndef prepare_features(df):\n    print(\"🧩 Preparing features...\")\n\n    feature_cols = [\n        'distance', 'relative_orientation', 'relative_velocity', 'relative_acceleration',\n        'COM_x', 'COM_y', 'COM_x_tgt', 'COM_y_tgt',\n        'mouse_id', 'mouse_id_target', 'arena_type', 'no_of_mice',\n        'mouse_strain', 'target_mouse_strain',\n        'mouse_sex', 'target_mouse_sex',\n        'mouse_age', 'target_mouse_age',\n        'mouse_color', 'target_mouse_color',\n        'mouse_condition', 'target_mouse_condition',\n        'mouse_condition_time_in_hours', 'target_mouse_condition_time_in_hours',\n        'mouse_condition_light_status', 'target_mouse_condition_light_status'\n    ]\n\n    keep_cols = [c for c in feature_cols if c in df.columns] + ['Target', 'lab_id', 'video_id']\n    if \"video_frame\" in df.columns:\n        keep_cols.append(\"video_frame\")\n\n    df = df[keep_cols].copy()\n\n    categorical_cols = [\n        'arena_type', 'mouse_strain', 'target_mouse_strain',\n        'mouse_sex', 'target_mouse_sex', 'mouse_age', 'target_mouse_age',\n        'mouse_color', 'target_mouse_color', 'mouse_condition', 'target_mouse_condition',\n        'mouse_condition_light_status', 'target_mouse_condition_light_status'\n    ]\n    categorical_cols = [c for c in categorical_cols if c in df.columns]\n\n    num_cols = [\n        'distance', 'relative_orientation', 'relative_velocity', 'relative_acceleration',\n        'COM_x', 'COM_y', 'COM_x_tgt', 'COM_y_tgt', 'no_of_mice',\n        'mouse_condition_time_in_hours', 'target_mouse_condition_time_in_hours'\n    ]\n    num_cols = [c for c in num_cols if c in df.columns]\n\n    # Fill NAs\n    for col in categorical_cols:\n        df[col] = df[col].fillna(\"unknown\").astype(str)\n    for col in num_cols:\n        df[col] = df[col].fillna(0)\n        df[col] = pd.to_numeric(df[col], downcast=\"float\")\n\n    print(\"🔢 Encoding categoricals...\")\n    le_dict = {}\n    for col in categorical_cols:\n        le = LabelEncoder()\n        df[col] = le.fit_transform(df[col])\n        le_dict[col] = le\n\n    scaler = StandardScaler()\n    scaled = scaler.fit_transform(df[num_cols])\n    df[num_cols] = scaled\n\n    gc.collect()\n    feature_cols_final = [c for c in df.columns if c not in ['Target', 'lab_id', 'video_id', 'video_frame']]\n    print(f\"✅ Features ready: {len(feature_cols_final)} cols, shape={df.shape}\")\n    return df, feature_cols_final, le_dict, scaler\n\n\n# ============================================================\n# 3️⃣ Base Training\n# ============================================================\ndef train_initial_model(df, feature_cols):\n    print(\"🚀 Training base RandomForest model...\")\n    X = df[feature_cols]\n    y = df[\"Target\"]\n\n    le_y = LabelEncoder()\n    y_enc = le_y.fit_transform(y)\n\n    X_train, X_val, y_train, y_val = train_test_split(X, y_enc, test_size=0.1, stratify=y_enc, random_state=42)\n    model = RandomForestClassifier(n_estimators=250, max_depth=18, n_jobs=-1, random_state=42)\n    model.fit(X_train, y_train)\n\n    preds = model.predict(X_val)\n    f1 = f1_score(y_val, preds, average=\"macro\")\n    print(f\"📈 Base F1: {f1:.4f}\")\n    print(classification_report(y_val, preds, target_names=le_y.classes_))\n    return model, le_y\n\n\n# ============================================================\n# 4️⃣ Semi-supervised Streaming Iterations\n# ============================================================\ndef semi_supervised_iterations_streaming(model, le_y, df_train, df_semi_meta, base_dir, feature_cols, le_dict, scaler,\n                                         rounds=3, threshold=0.9):\n    print(f\"🔁 Starting semi-supervised refinement ({rounds} rounds, threshold={threshold})\")\n\n    for epoch in range(rounds):\n        print(f\"\\n🌀 Round {epoch+1}/{rounds}\")\n        pseudo_list = []\n        total_added = 0\n\n        for _, row in tqdm(df_semi_meta.iterrows(), total=len(df_semi_meta), desc=\"Video loop\"):\n            \n            lab_id, vid = row[\"lab_id\"], row[\"video_id\"]\n            fpath = f\"{base_dir}/{lab_id}/{vid}.parquet\"\n\n            if not os.path.exists(fpath):\n                continue\n\n            try:\n                df_vid = pd.read_parquet(fpath)\n                df_vid[\"lab_id\"] = lab_id\n                df_vid[\"video_id\"] = vid\n\n                # sample 10%\n                x = int(df_vid.shape[0] * 0.1)\n                df_vid = df_vid.head(x)\n\n                df_vid = preprocess_like_training(df_vid, feature_cols, le_dict, scaler)\n\n                X_vid = df_vid[feature_cols]\n                probs = model.predict_proba(X_vid)\n                conf = np.max(probs, axis=1)\n                preds = np.argmax(probs, axis=1)\n                mask = conf >= threshold\n\n                if mask.sum() == 0:\n                    continue\n\n                df_high = df_vid.loc[mask, feature_cols + [\"lab_id\", \"video_id\", \"video_frame\"]].copy()\n                df_high[\"Target\"] = le_y.inverse_transform(preds[mask])\n                df_high[\"Confidence\"] = conf[mask]\n\n                pseudo_list.append(df_high)\n                total_added += len(df_high)\n                del df_vid, df_high, X_vid, conf, preds, probs, mask\n                gc.collect()\n\n            except Exception as e:\n                print(f\"⚠️ {vid} error: {e}\")\n                continue\n\n        if not pseudo_list:\n            print(\"⚠️ No confident pseudo-labels this round — stopping early.\")\n            break\n\n        pseudo_df = pd.concat(pseudo_list, ignore_index=True)\n        df_train = pd.concat([df_train, pseudo_df], ignore_index=True)\n        del pseudo_list, pseudo_df; gc.collect()\n\n        print(f\"✅ Round {epoch+1}: added {total_added:,} pseudo-labeled samples.\")\n        print(f\"🔁 Retraining model on {len(df_train):,} samples...\")\n\n        X_train = df_train[feature_cols]\n        y_train = le_y.transform(df_train[\"Target\"])\n        model.fit(X_train, y_train)\n\n    print(\"🏁 Semi-supervised training complete.\")\n    return model, df_train\n\ndef preprocess_like_training(df_vid, feature_cols, le_dict, scaler):\n    \"\"\"\n    Apply the same preprocessing used in training (in-place-ish).\n    Returns a dataframe where categorical columns are encoded, numeric cols are scaled,\n    and all feature_cols are present.\n    \"\"\"\n    df = df_vid.copy()\n\n    categorical_cols = list(le_dict.keys())\n    # scaler.feature_names_in_ is an array of the numeric columns used at training\n    num_cols = list(getattr(scaler, \"feature_names_in_\", []))\n\n    # Ensure frame_id exists for traceability\n    if \"video_frame\" not in df.columns:\n        df[\"video_frame\"] = np.arange(len(df), dtype=np.int64)\n\n    # Handle categorical columns robustly\n    for col in categorical_cols:\n        if col in df.columns:\n            df[col] = df[col].fillna(\"unknown\").astype(str)\n            # safer transform: try/except to handle unseen labels\n            le = le_dict[col]\n            def safe_transform(v):\n                try:\n                    return le.transform([v])[0]\n                except Exception:\n                    return -1\n            df[col] = df[col].map(safe_transform)\n        else:\n            df[col] = -1\n\n    # Ensure numeric columns exist and are numeric\n    for col in num_cols:\n        if col in df.columns:\n            df[col] = pd.to_numeric(df[col].fillna(0), errors=\"coerce\").fillna(0)\n        else:\n            df[col] = 0.0\n\n    # If there are no numeric columns scaler may fail; handle that\n    if len(num_cols) > 0:\n        df[num_cols] = scaler.transform(df[num_cols])\n\n    # Ensure all feature_cols exist\n    for c in feature_cols:\n        if c not in df.columns:\n            df[c] = 0\n\n    # Keep original ordering (optional)\n    return df\n\n\n# def predict_and_score_unseen(model, le_y, df, base_dir, feature_cols, le_dict, scaler, sample_frac=None):\n#     \"\"\"\n#     Streams through unseen video parquet files listed in df_meta (columns: lab_id, video_id),\n#     applies the exact same preprocessing as training via preprocess_like_training(),\n#     runs model.predict_proba, stores Predicted + Confidence and preserves frame_id in final output.\n\n#     Args:\n#       - model, le_y, feature_cols, le_dict, scaler: from training pipeline\n#       - df_meta: DataFrame with columns \"lab_id\" and \"video_id\" (one row per video)\n#       - base_dir: folder root where lab_id/video_id.parquet live\n#       - sample_frac: optional float in (0,1] to sample that fraction of each video frames (memory)\n#     Returns:\n#       - preds_df: DataFrame with columns [lab_id, video_id, frame_id, Predicted, Confidence, (Target if present)]\n#       - f1: macro F1 score if Target column present in preds_df else None\n#     \"\"\"\n#     results = []\n#     # for _, row in tqdm(df_meta.iterrows(), total=len(df_meta), desc=\"Unseen scoring\"):\n#     # lab_id, vid = row[\"lab_id\"], row[\"video_id\"]\n#     # fpath = f\"{base_dir}/{lab_id}/{vid}.parquet\"\n#     # if not os.path.exists(fpath):\n#     #     print(f\"⚠️ Missing file:\", fpath)\n#     #     continue\n\n#     try:\n#         # df_vid = pd.read_parquet(fpath)\n#         # attach ids\n#         df_vid = df\n#         df_vid[\"lab_id\"] = lab_id\n#         df_vid[\"video_id\"] = vid\n\n#         # optional sampling to reduce memory/compute\n#         if sample_frac is not None and 0 < sample_frac < 1:\n#             # sample preserves index; reset index afterwards for safe numeric ops\n#             df_vid = df_vid.sample(frac=sample_frac, random_state=42).reset_index(drop=True)\n\n#         # Apply identical preprocessing\n#         df_proc = preprocess_like_training(df_vid, feature_cols, le_dict, scaler)\n\n#         # Build feature matrix and predict\n#         X_vid = df_proc[feature_cols]\n#         probs = model.predict_proba(X_vid)\n#         preds_idx = np.argmax(probs, axis=1)\n#         conf = np.max(probs, axis=1)\n\n#         # Map back to labels\n#         preds_label = le_y.inverse_transform(preds_idx)\n\n#         # Prepare output: keep only needed columns (preserve frame_id)\n#         out = pd.DataFrame({\n#             \"lab_id\": df_proc[\"lab_id\"].values,\n#             \"video_id\": df_proc[\"video_id\"].values,\n#             \"frame_id\": df_proc[\"video_frame\"].values,\n#             \"Mouse_id\" : df_proc[\"mouse_id\"].values,\n#             \"Target_id\" : df_proc[\"mouse_id_target\"].values,\n#             \"Predicted\": preds_label,\n#             \"Confidence\": conf\n#         })\n\n#         # include Target if present in original df_vid (before preprocess)\n#         if \"Target\" in df_vid.columns:\n#             # align by index: df_proc was copy and may have been sampled; df_vid reset_index when sampled\n#             # use df_proc index to fetch Target if it existed originally; safer to merge by frame_id if needed\n#             if \"Target\" in df_proc.columns:\n#                 out[\"Target\"] = df_proc[\"Target\"].values\n#             else:\n#                 # if preprocess removed Target, try to get from original df_vid by frame_id\n#                 if \"video_frame\" in df_vid.columns:\n#                     tgt_map = df_vid.set_index(\"video_frame\")[\"Target\"] if \"Target\" in df_vid.columns else None\n#                     if tgt_map is not None:\n#                         out[\"Target\"] = out[\"video_frame\"].map(tgt_map).values\n#                     else:\n#                         out[\"Target\"] = np.nan\n#                 else:\n#                     out[\"Target\"] = np.nan\n\n#         results.append(out)\n\n#         # cleanup per-video\n#         del df_vid, df_proc, X_vid, probs, preds_idx, conf, preds_label, out\n#         gc.collect()\n\n#     except Exception as e:\n#         print(f\"❌ Error scoring video {vid}: {e}\")\n#         continue\n\n#     preds_df = pd.concat(results, ignore_index=True) if results else pd.DataFrame()\n#     print(f\"✅ Generated predictions for {len(preds_df):,} frames.\")\n\n#     # compute F1 if targets available\n#     if \"Target\" in preds_df.columns and preds_df[\"Target\"].notna().any():\n#         f1 = f1_score(preds_df[\"Target\"].fillna(\"None\"), preds_df[\"Predicted\"].fillna(\"None\"),\n#                       average=\"macro\", zero_division=0)\n#         print(f\"🏁 F1 on unseen (if target present): {f1:.4f}\")\n#     else:\n#         f1 = None\n#         print(\"⚠️ No Target found in unseen data; F1 not computed.\")\n\n#     return preds_df, f1\n\n\n# ============================================================\n# 6️⃣ Full Orchestrator\n# ============================================================\ndef run_two_stage_pipeline(df_train, df_semi, base_dir):\n    df_train_full = load_behavior_data(df_train, base_dir)\n    # df_train_full = df\n    df_train_full, feature_cols, le_dict, scaler = prepare_features(df_train_full)\n    model, le_y = train_initial_model(df_train_full, feature_cols)\n\n    model, trained_df = semi_supervised_iterations_streaming(\n        model, le_y, df_train_full, df_semi, base_dir,\n        feature_cols, le_dict, scaler, rounds=3, threshold=1\n    )\n\n    # preds_df, f1 = predict_and_score_unseen(model, le_y, df_unseen, base_dir, feature_cols, le_dict, scaler)\n\n    print(\"✅ Full pipeline complete.\")\n    return model, trained_df, le_y, feature_cols, le_dict, scaler\n\n\ntrain = pd.read_csv('/kaggle/input/MABe-mouse-behavior-detection/train.csv')\ntrain.drop(columns=['mouse1_id', 'mouse2_id', 'mouse3_id', 'mouse4_id'], inplace=True)\n\ndf_semi = train[train['behaviors_labeled'].isna()]\ndf_train = train[train['behaviors_labeled'].notna()]\ndf_semi = df_semi.sample(1)\n\nbase_dir = \"/kaggle/input/mabe-dataset/Dataset\"\nmodel, trained_df, le_y, feature_cols, le_dict, scaler = run_two_stage_pipeline(df_train, df_semi, base_dir)\n\n\n\ndef predict_and_score_unseen(\n    model,\n    le_y,\n    df,\n    base_dir,\n    feature_cols,\n    le_dict,\n    scaler,\n    sample_frac=None\n):\n\n    \"\"\"\n    df → single dataframe containing unseen data for one video.\n    Must include frame_id, mouse_id, mouse_id_target and (optionally) Target.\n    \"\"\"\n\n    # -----------------------------\n    # 1️⃣ Copy and preserve essentials\n    # -----------------------------\n    df_vid = df.copy()\n\n    # Preserve raw frame_id\n    if \"frame_id\" not in df_vid.columns:\n        raise ValueError(\"❌ df must contain frame_id column.\")\n\n    # optional sampling\n    if sample_frac is not None and 0 < sample_frac < 1:\n        df_vid = df_vid.sample(frac=sample_frac, random_state=42).reset_index(drop=True)\n\n    # ---------------------------------\n    # 2️⃣ Apply identical preprocessing\n    # ---------------------------------\n    df_proc = preprocess_like_training(df_vid, feature_cols, le_dict, scaler)\n\n    # ---------------------------------\n    # 3️⃣ Predict on features\n    # ---------------------------------\n    X = df_proc[feature_cols]\n    probs = model.predict_proba(X)\n    pred_idx = np.argmax(probs, axis=1)\n    conf = np.max(probs, axis=1)\n    pred_lbl = le_y.inverse_transform(pred_idx)\n\n    # -----------------------------------------------\n    # 4️⃣ Build final output dataframe (with frame_id)\n    # -----------------------------------------------\n    preds_df = pd.DataFrame({\n        \"lab_id\": df_vid.get(\"lab_id\", None),\n        \"video_id\": df_vid.get(\"video_id\", None),\n        \"frame_id\": df_vid[\"frame_id\"].values,\n        \"Mouse_id\": df_vid[\"mouse_id\"].values,\n        \"Target_id\": df_vid[\"mouse_id_target\"].values,\n        \"Predicted\": pred_lbl,\n        \"Confidence\": conf\n    })\n\n    # ---------------------------------\n    # 5️⃣ Add Target if exists\n    # ---------------------------------\n    if \"Target\" in df_vid.columns:\n        preds_df[\"Target\"] = df_vid[\"Target\"].values\n    else:\n        preds_df[\"Target\"] = None\n\n    # ---------------------------------\n    # 6️⃣ Compute F1 if Target exists\n    # ---------------------------------\n    if preds_df[\"Target\"].notna().any():\n        # Fill missing as string \"None\"\n        t = preds_df[\"Target\"].fillna(\"None\")\n        p = preds_df[\"Predicted\"].fillna(\"None\")\n\n        f1 = f1_score(t, p, average=\"macro\", zero_division=0)\n    else:\n        f1 = None\n\n    return preds_df, f1\n\n\nfinal_df.rename(columns = {'video_frame':'frame_id'}, inplace =True)\npreds_df, f1 = predict_and_score_unseen(\n    model=model,\n    le_y=le_y,\n    df=final_df,\n    base_dir=base_dir,\n    feature_cols=feature_cols,\n    le_dict=le_dict,\n    scaler=scaler\n)\n\n\n\npreds_conf = preds_df[preds_df['Confidence']>0.5]\n\n\n\ndef convert_frames_to_segments(df):\n    \"\"\"\n    Convert frame-level predictions to action segments.\n    Also convert numeric Mouse_id and Target_id into mouse1, mouse2, ...\n    \"\"\"\n\n    df = df.sort_values([\"video_id\", \"Mouse_id\", \"Target_id\", \"frame_id\"]).reset_index(drop=True)\n\n    segments = []\n\n    for (vid, agent, target, action), group in df.groupby(\n        [\"video_id\", \"Mouse_id\", \"Target_id\", \"Predicted\"]\n    ):\n        group = group.sort_values(\"frame_id\")\n\n        # Detect continuous frame sequences\n        diffs = group[\"frame_id\"].diff().fillna(1)\n        segment_id = (diffs != 1).cumsum()\n\n        for _, seg_df in group.groupby(segment_id):\n            start_f = int(seg_df[\"frame_id\"].iloc[0])\n            stop_f  = int(seg_df[\"frame_id\"].iloc[-1])\n\n            segments.append({\n                \"video_id\": vid,\n                \"agent_id\": f\"mouse{int(agent)}\",\n                \"target_id\": f\"mouse{int(target)}\",\n                \"action\": action,\n                \"start_frame\": start_f,\n                \"stop_frame\": stop_f,\n            })\n\n    seg_df = pd.DataFrame(segments).reset_index().rename(columns={\"index\": \"row_id\"})\n    return seg_df\n\n\npredictions = convert_frames_to_segments(preds_conf)\n\npredictions.to_csv('/kaggle/working/submission.csv')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-15T07:24:48.541795Z","iopub.execute_input":"2025-11-15T07:24:48.542280Z","iopub.status.idle":"2025-11-15T07:24:54.743232Z","shell.execute_reply.started":"2025-11-15T07:24:48.542251Z","shell.execute_reply":"2025-11-15T07:24:54.741945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"predictions.to_csv('/kaggle/working/submission.csv',index = False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-15T07:25:46.010557Z","iopub.execute_input":"2025-11-15T07:25:46.011064Z","iopub.status.idle":"2025-11-15T07:25:46.050275Z","shell.execute_reply.started":"2025-11-15T07:25:46.011017Z","shell.execute_reply":"2025-11-15T07:25:46.049080Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}