{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":262477103,"sourceType":"kernelVersion"},{"sourceId":279806245,"sourceType":"kernelVersion"},{"sourceId":281545678,"sourceType":"kernelVersion"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MABe Challenge - XGBoost inference Notebook","metadata":{}},{"cell_type":"markdown","source":"📝 **Note:** Please note that comments and explanations are in Japanese. However, I've made an effort to write clear, self-explanatory code that should be accessible to non-Japanese speakers as well.","metadata":{}},{"cell_type":"markdown","source":"## training notebook: \nhttps://www.kaggle.com/code/hutch1221/mabe-starter-train-ja/notebook","metadata":{}},{"cell_type":"code","source":"!pip install -q --no-index --find-links=/kaggle/input/mabe-package xgboost==3.1.1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:10:47.276234Z","iopub.execute_input":"2025-11-25T06:10:47.276447Z","iopub.status.idle":"2025-11-25T06:10:58.945652Z","shell.execute_reply.started":"2025-11-25T06:10:47.276426Z","shell.execute_reply":"2025-11-25T06:10:58.944506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!cp /kaggle/input/mabe-starter-train-ja/self_features.py .\n!cp /kaggle/input/mabe-starter-train-ja/pair_features.py .\n!cp /kaggle/input/mabe-starter-train-ja/robustify.py .\n!cp -r /kaggle/input/mabe-starter-train-ja/results .","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:10:58.948329Z","iopub.execute_input":"2025-11-25T06:10:58.948716Z","iopub.status.idle":"2025-11-25T06:11:12.761563Z","shell.execute_reply.started":"2025-11-25T06:10:58.948672Z","shell.execute_reply":"2025-11-25T06:11:12.760112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import gc\nimport itertools\nimport re\nimport sys\nfrom collections import defaultdict\nfrom pathlib import Path\n\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport xgboost as xgb\nfrom tqdm.auto import tqdm\n\nsys.path.append(\"/kaggle/usr/lib/mabe-f-beta\")\nfrom metric import score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:12.777434Z","iopub.execute_input":"2025-11-25T06:11:12.777876Z","iopub.status.idle":"2025-11-25T06:11:14.699477Z","shell.execute_reply.started":"2025-11-25T06:11:12.777828Z","shell.execute_reply":"2025-11-25T06:11:14.698565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# const\nINPUT_DIR = Path(\"/kaggle/input/MABe-mouse-behavior-detection\")\nTRAIN_TRACKING_DIR = INPUT_DIR / \"train_tracking\"\nTRAIN_ANNOTATION_DIR = INPUT_DIR / \"train_annotation\"\nTEST_TRACKING_DIR = INPUT_DIR / \"test_tracking\"\n\nWORKING_DIR = Path(\"/kaggle/working\")\n\nINDEX_COLS = [\n    \"video_id\",\n    \"agent_mouse_id\",\n    \"target_mouse_id\",\n    \"video_frame\",\n]\n\nBODY_PARTS = [\n    \"ear_left\",\n    \"ear_right\",\n    \"nose\",\n    \"neck\",\n    \"body_center\",\n    \"lateral_left\",\n    \"lateral_right\",\n    \"hip_left\",\n    \"hip_right\",\n    \"tail_base\",\n    \"tail_tip\",\n]\n\nSELF_BEHAVIORS = [\n    \"biteobject\",\n    \"climb\",\n    \"dig\",\n    \"exploreobject\",\n    \"freeze\",\n    \"genitalgroom\",\n    \"huddle\",\n    \"rear\",\n    \"rest\",\n    \"run\",\n    \"selfgroom\",\n]\n\nPAIR_BEHAVIORS = [\n    \"allogroom\",\n    \"approach\",\n    \"attack\",\n    \"attemptmount\",\n    \"avoid\",\n    \"chase\",\n    \"chaseattack\",\n    \"defend\",\n    \"disengage\",\n    \"dominance\",\n    \"dominancegroom\",\n    \"dominancemount\",\n    \"ejaculate\",\n    \"escape\",\n    \"flinch\",\n    \"follow\",\n    \"intromit\",\n    \"mount\",\n    \"reciprocalsniff\",\n    \"shepherd\",\n    \"sniff\",\n    \"sniffbody\",\n    \"sniffface\",\n    \"sniffgenital\",\n    \"submit\",\n    \"tussle\",\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:14.700546Z","iopub.execute_input":"2025-11-25T06:11:14.701007Z","iopub.status.idle":"2025-11-25T06:11:14.708290Z","shell.execute_reply.started":"2025-11-25T06:11:14.700976Z","shell.execute_reply":"2025-11-25T06:11:14.706886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# read data\ntest_dataframe = pl.read_csv(INPUT_DIR / \"test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:14.709357Z","iopub.execute_input":"2025-11-25T06:11:14.709708Z","iopub.status.idle":"2025-11-25T06:11:14.770483Z","shell.execute_reply.started":"2025-11-25T06:11:14.709667Z","shell.execute_reply":"2025-11-25T06:11:14.769816Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# preprocess behavior labels\ntest_behavior_dataframe = (\n    test_dataframe.filter(pl.col(\"behaviors_labeled\").is_not_null())\n    .select(\n        pl.col(\"lab_id\"),\n        pl.col(\"video_id\"),\n        pl.col(\"behaviors_labeled\").map_elements(eval, return_dtype=pl.List(pl.Utf8)).alias(\"behaviors_labeled_list\"),\n    )\n    .explode(\"behaviors_labeled_list\")\n    .rename({\"behaviors_labeled_list\": \"behaviors_labeled_element\"})\n    .select(\n        pl.col(\"lab_id\"),\n        pl.col(\"video_id\"),\n        pl.col(\"behaviors_labeled_element\").str.split(\",\").list[0].str.replace_all(\"'\", \"\").alias(\"agent\"),\n        pl.col(\"behaviors_labeled_element\").str.split(\",\").list[1].str.replace_all(\"'\", \"\").alias(\"target\"),\n        pl.col(\"behaviors_labeled_element\").str.split(\",\").list[2].str.replace_all(\"'\", \"\").alias(\"behavior\"),\n    )\n)\n\ntest_self_behavior_dataframe = test_behavior_dataframe.filter(pl.col(\"behavior\").is_in(SELF_BEHAVIORS))\ntest_pair_behavior_dataframe = test_behavior_dataframe.filter(pl.col(\"behavior\").is_in(PAIR_BEHAVIORS))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:14.772668Z","iopub.execute_input":"2025-11-25T06:11:14.772907Z","iopub.status.idle":"2025-11-25T06:11:14.924440Z","shell.execute_reply.started":"2025-11-25T06:11:14.772888Z","shell.execute_reply":"2025-11-25T06:11:14.923746Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%run -i self_features.py\n%run -i pair_features.py\n%run -i robustify.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:14.925246Z","iopub.execute_input":"2025-11-25T06:11:14.925489Z","iopub.status.idle":"2025-11-25T06:11:14.937149Z","shell.execute_reply.started":"2025-11-25T06:11:14.925468Z","shell.execute_reply":"2025-11-25T06:11:14.936161Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(WORKING_DIR / \"self_features\").mkdir(exist_ok=True, parents=True)\n(WORKING_DIR / \"pair_features\").mkdir(exist_ok=True, parents=True)\n\nrows = test_dataframe.rows(named=True)\n\nfor row in tqdm(rows, total=len(rows)):\n    lab_id = row[\"lab_id\"]\n    video_id = row[\"video_id\"]\n\n    tracking_path = TEST_TRACKING_DIR / f\"{lab_id}/{video_id}.parquet\"\n    tracking = pl.read_parquet(tracking_path)\n\n    self_features = make_self_features(metadata=row, tracking=tracking)\n    pair_features = make_pair_features(metadata=row, tracking=tracking)\n\n    self_features.write_parquet(WORKING_DIR / \"self_features\" / f\"{video_id}.parquet\")\n    pair_features.write_parquet(WORKING_DIR / \"pair_features\" / f\"{video_id}.parquet\")\n\n    del self_features, pair_features\n    gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:14.938231Z","iopub.execute_input":"2025-11-25T06:11:14.938578Z","iopub.status.idle":"2025-11-25T06:11:17.322433Z","shell.execute_reply.started":"2025-11-25T06:11:14.938544Z","shell.execute_reply":"2025-11-25T06:11:17.321288Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# submissionデータ作成","metadata":{}},{"cell_type":"code","source":"# 各グループ（lab_id, video_id, agent, target の組み合わせ）ごとの予測結果を格納するリスト\ngroup_submissions = []\n\n# テストデータを lab_id, video_id, agent, target でグループ化\n# maintain_order=True で元の順序を保持\ngroups = list(test_behavior_dataframe.group_by(\"lab_id\", \"video_id\", \"agent\", \"target\", maintain_order=True))\n\n# 各グループに対して順番に処理を実行（進捗バーを表示）\nfor (lab_id, video_id, agent, target), group in tqdm(groups, total=len(list(groups))):\n    # agent（行動を起こすマウス）のID を抽出\n    # 例: \"mouse1\" → 1\n    agent_mouse_id = int(re.search(r\"mouse(\\d+)\", agent).group(1))\n    \n    # target（行動の対象）のID を抽出\n    # \"self\"（自己行動）の場合は -1、それ以外はマウスIDを抽出\n    # 例: \"mouse2\" → 2, \"self\" → -1\n    target_mouse_id = -1 if target == \"self\" else int(re.search(r\"mouse(\\d+)\", target).group(1))\n\n    # ===== 特徴量データの読み込み =====\n    if target == \"self\":\n        # 自己行動（rear など）の場合: self_features ディレクトリから読み込み\n        \n        # インデックス列（video_id, agent_mouse_id, video_frame など）を読み込み\n        index = (\n            pl.scan_parquet(WORKING_DIR / \"self_features\" / f\"{video_id}.parquet\")\n            .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id))  # 対象マウスでフィルタ\n            .select(INDEX_COLS)  # インデックス列のみ選択\n            .collect()  # 遅延評価を実行してデータを取得\n        )\n        \n        # 特徴量列（速度、距離、角度など）を読み込み\n        feature = (\n            pl.scan_parquet(WORKING_DIR / \"self_features\" / f\"{video_id}.parquet\")\n            .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id))\n            .select(pl.exclude(INDEX_COLS))  # インデックス列以外を選択\n            .collect()\n        )\n    else:\n        # ペア行動（attack, chase など）の場合: pair_features ディレクトリから読み込み\n        \n        # インデックス列を読み込み（agent と target の両方でフィルタ）\n        index = (\n            pl.scan_parquet(WORKING_DIR / \"pair_features\" / f\"{video_id}.parquet\")\n            .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id) & (pl.col(\"target_mouse_id\") == target_mouse_id))\n            .select(INDEX_COLS)\n            .collect()\n        )\n        \n        # 特徴量列を読み込み\n        feature = (\n            pl.scan_parquet(WORKING_DIR / \"pair_features\" / f\"{video_id}.parquet\")\n            .filter((pl.col(\"agent_mouse_id\") == agent_mouse_id) & (pl.col(\"target_mouse_id\") == target_mouse_id))\n            .select(pl.exclude(INDEX_COLS))\n            .collect()\n        )\n\n    # 予測結果を格納する DataFrame を作成（インデックス列のコピー）\n    prediction_dataframe = index.clone()\n\n    # ===== 各行動（behavior）に対して予測を実行 =====\n    for row in group.rows(named=True):\n        behavior = row[\"behavior\"]  # 現在の行動名（例: \"attack\", \"rear\"）\n\n        # 各 fold（交差検証の分割）の予測結果を格納するリスト\n        predictions = []  # 予測確率\n        prediction_labels = []  # 予測ラベル（閾値で 0/1 に変換したもの）\n\n        # 保存された fold ディレクトリを取得\n        # 例: results/AdaptableSnail/attack/fold_0, fold_1, fold_2\n        fold_dirs = list((WORKING_DIR / \"results\" / lab_id / behavior).glob(\"fold_*\"))\n        if not fold_dirs:\n            # 訓練されたモデルが見つからない場合はスキップ\n            continue\n\n        # 各 fold のモデルで予測を実行\n        for fold_dir in fold_dirs:\n            # 保存された最適閾値を読み込み\n            with open(fold_dir / \"threshold.txt\", \"r\") as f:\n                threshold = float(f.read().strip())\n            \n            # XGBoost モデルを読み込み\n            model = xgb.Booster(model_file=fold_dir / \"model.json\")\n            \n            # 特徴量を XGBoost の入力形式（DMatrix）に変換\n            dtest = xgb.DMatrix(feature, feature_names=feature.columns)\n            \n            # モデルで予測を実行（確率値を取得）\n            fold_predictions = model.predict(dtest)\n            \n            # 予測確率を保存\n            predictions.append(fold_predictions)\n            \n            # 閾値を適用してラベル化（1: 行動あり, 0: 行動なし）\n            prediction_labels.append((fold_predictions >= threshold).astype(np.int8))\n\n        # 予測結果を DataFrame に追加\n        # 各 fold の「予測確率 × 予測ラベル」を列として追加\n        # （ラベルが 0 の場合は確率も 0 になり、1 の場合は確率がそのまま残る）\n        prediction_dataframe = prediction_dataframe.with_columns(\n            *[\n                pl.Series(name=f\"{behavior}_{fold}\", values=predictions[fold] * prediction_labels[fold], dtype=pl.Float32)\n                for fold in range(len(fold_dirs))\n            ]\n        )\n\n    # ===== 最も確率が高い行動を選択 =====\n    \n    # インデックス列以外の列名を取得（各行動の予測列）\n    cols = prediction_dataframe.select(pl.exclude(INDEX_COLS)).columns\n    if not cols:\n        # 予測列が 1 つもない場合は警告を表示してスキップ\n        tqdm.write(f\"Warning: No predictions found for {lab_id}, {video_id}, {agent}, {target}\")\n        continue\n\n    # 各フレームで最も確率が高い行動を選択\n    prediction_labels_dataframe = prediction_dataframe.with_columns(\n        pl.struct(pl.col(cols))  # 全予測列を構造体にまとめる\n        .map_elements(\n            # 各行（フレーム）に対して以下の処理を実行:\n            # - すべての予測値が 0 なら \"none\"（行動なし）\n            # - それ以外は最大値を持つ行動名を返す\n            lambda row: \"none\" if sum(row.values()) == 0 else (cols[np.argmax(list(row.values()))]).split(\"_\")[0],\n            return_dtype=pl.String,\n        )\n        .alias(\"prediction\")  # 新しい列名を \"prediction\" とする\n    ).select(INDEX_COLS + [\"prediction\"])  # インデックス列と予測列のみ選択\n\n    # ===== 連続する同じ行動をイベントにまとめる =====\n    \n    group_submission = (\n        prediction_labels_dataframe\n        .filter((pl.col(\"prediction\") != pl.col(\"prediction\").shift(1)))  # 行動が変化したフレームのみ残す\n        .with_columns(pl.col(\"video_frame\").shift(-1).alias(\"stop_frame\"))  # 次の変化点を終了フレームとする\n        .filter(pl.col(\"prediction\") != \"none\")  # \"none\"（行動なし）を除外\n        .select(\n            # 提出形式に合わせて列を選択・変換\n            pl.col(\"video_id\"),\n            (\"mouse\" + pl.col(\"agent_mouse_id\").cast(str)).alias(\"agent_id\"),  # 例: 1 → \"mouse1\"\n            pl.when(pl.col(\"target_mouse_id\") == -1)  # target_mouse_id が -1 なら\n            .then(pl.lit(\"self\"))  # \"self\" に変換\n            .otherwise(\"mouse\" + pl.col(\"target_mouse_id\").cast(str))  # それ以外は \"mouseN\"\n            .alias(\"target_id\"),\n            pl.col(\"prediction\").alias(\"action\"),  # 行動名\n            pl.col(\"video_frame\").alias(\"start_frame\"),  # 開始フレーム\n            pl.col(\"stop_frame\"),  # 終了フレーム\n        )\n    )\n\n    # このグループの提出データをリストに追加\n    group_submissions.append(group_submission)\n\n# ===== 全グループの予測結果を結合 =====\n\n# 全グループの提出データを縦方向に結合\nsubmission = pl.concat(group_submissions, how=\"vertical\").sort(\n    \"video_id\",\n    \"agent_id\",\n    \"target_id\",\n    \"action\",\n    \"start_frame\",\n    \"stop_frame\",\n)\n\n# 提出データの堅牢化処理（重複削除、フレームの修正など）\nsubmission = robustify(submission, test_dataframe, train_test=\"test\")\n\n# 行番号（row_id）を追加して CSV ファイルとして保存\nsubmission.with_row_index(\"row_id\").write_csv(WORKING_DIR / \"submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:11:17.323571Z","iopub.execute_input":"2025-11-25T06:11:17.323877Z","iopub.status.idle":"2025-11-25T06:12:00.491980Z","shell.execute_reply.started":"2025-11-25T06:11:17.323844Z","shell.execute_reply":"2025-11-25T06:12:00.491069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!head submission.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-25T06:12:00.493050Z","iopub.execute_input":"2025-11-25T06:12:00.493680Z","iopub.status.idle":"2025-11-25T06:12:00.638771Z","shell.execute_reply.started":"2025-11-25T06:12:00.493646Z","shell.execute_reply":"2025-11-25T06:12:00.637905Z"}},"outputs":[],"execution_count":null}]}