{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13433768,"sourceType":"datasetVersion","datasetId":8526611}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport math\nimport numpy as np\nfrom pathlib import Path\nfrom typing import Literal, Union, Optional\nfrom tqdm.auto import tqdm\nimport itertools\nimport os\nimport json\nimport yaml\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport torch\ntorch.backends.cudnn.benchmark = True\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport numpy as np\nfrom sklearn.metrics import classification_report\nfrom typing import List, Tuple, Dict\nfrom collections import Counter","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:27.903983Z","iopub.execute_input":"2025-10-19T16:37:27.904533Z","iopub.status.idle":"2025-10-19T16:37:27.912830Z","shell.execute_reply.started":"2025-10-19T16:37:27.904496Z","shell.execute_reply":"2025-10-19T16:37:27.911535Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Helper","metadata":{}},{"cell_type":"code","source":"class RawTableLoader:    \n    def __init__(self, root_path: Path):\n        self.root_path = root_path\n        self.train_path = root_path / \"train.csv\"\n        self.test_path = root_path / \"test.csv\"\n        self.submission_path = root_path / \"sample_submission.csv\"\n        self.drop_body_parts = [\n            'headpiece_bottombackleft', 'headpiece_bottombackright', 'headpiece_bottomfrontleft', 'headpiece_bottomfrontright', \n            'headpiece_topbackleft', 'headpiece_topbackright', 'headpiece_topfrontleft', 'headpiece_topfrontright', \n            'spine_1', 'spine_2',\n            'tail_middle_1', 'tail_middle_2', 'tail_midpoint', \"tail_tip\", \"neck\"\n        ]\n\n    def read_train(self):\n        df = pd.read_csv(self.train_path)\n        return df\n\n    def read_test(self):\n        return pd.read_csv(self.test_path)\n\n    def read_submission(self):\n        return pd.read_csv(self.submission_path)\n\n    def read_tracking(self, data_type: Literal[\"train\", \"test\"], lab_id, video_id):\n        track = pd.read_parquet(self.root_path / f\"{data_type}_tracking\" / lab_id / f\"{video_id}.parquet\")\n        track = track[~track[\"bodypart\"].isin(self.drop_body_parts)]\n        track_piv = track.pivot_table(index=[\"video_frame\"], columns=[\"mouse_id\", \"bodypart\"], values=[\"x\", \"y\"])\n        track_piv = track_piv.reorder_levels([1,0,2], axis=1)\n\n        full_range = range(track_piv.index.min(), track_piv.index.max() + 1)\n        track_piv = track_piv.reindex(full_range)\n        \n        return track_piv\n\n    def read_annotation(self, data_type: Literal[\"train\"], lab_id, video_id):\n        return pd.read_parquet(self.root_path / f\"{data_type}_annotation\" / lab_id / f\"{video_id}.parquet\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:27.926320Z","iopub.execute_input":"2025-10-19T16:37:27.926663Z","iopub.status.idle":"2025-10-19T16:37:27.973499Z","shell.execute_reply.started":"2025-10-19T16:37:27.926640Z","shell.execute_reply":"2025-10-19T16:37:27.972425Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def interpolate_missing_values(data: pd.DataFrame, max_gap: int = 10) -> pd.DataFrame:\n    \"\"\"\n    時系列データの欠損値を線形補間で補完する\n    \n    Parameters:\n    -----------\n    data : pd.DataFrame\n        時系列データ（各列が変数、各行が時点）\n    max_gap : int, default=10\n        補完する最大連続欠損フレーム数\n    \n    Returns:\n    --------\n    pd.DataFrame\n        補完後のデータフレーム\n    \"\"\"\n    result = data.copy()\n    \n    for column in result.columns:\n        result[column] = interpolate_series(result[column], max_gap)\n    \n    return result\n\ndef interpolate_series(series: pd.Series, max_gap: int = 10) -> pd.Series:\n    \"\"\"\n    単一の時系列に対する欠損値補間\n    \n    Parameters:\n    -----------\n    series : pd.Series\n        時系列データ\n    max_gap : int\n        補完する最大連続欠損フレーム数\n    \n    Returns:\n    --------\n    pd.Series\n        補間後の時系列データ\n    \"\"\"\n    result = series.copy()\n    is_null = result.isnull()\n    \n    if not is_null.any():\n        return result\n    \n    # 連続する欠損区間を特定\n    null_groups = identify_null_groups(is_null)\n    \n    for start_idx, end_idx in null_groups:\n        gap_length = end_idx - start_idx + 1\n        \n        # 指定フレーム数以内の欠損のみ補完\n        if gap_length <= max_gap:\n            # 前の有効値を取得\n            before_val = get_last_valid_before(result, start_idx)\n            # 後の有効値を取得\n            after_val = get_first_valid_after(result, end_idx)\n            \n            if before_val is not None and after_val is not None:\n                # 線形補間を実行\n                interpolated_values = linear_interpolate(\n                    before_val, after_val, gap_length\n                )\n                result.iloc[start_idx:end_idx+1] = interpolated_values\n    \n    return result\n\ndef identify_null_groups(is_null: pd.Series) -> list:\n    \"\"\"\n    連続する欠損値の区間を特定\n    \n    Returns:\n    --------\n    list of tuple\n        (開始インデックス, 終了インデックス) のリスト\n    \"\"\"\n    groups = []\n    start = None\n    \n    for i, null_val in enumerate(is_null):\n        if null_val and start is None:\n            start = i\n        elif not null_val and start is not None:\n            groups.append((start, i - 1))\n            start = None\n    \n    # 最後まで欠損が続く場合\n    if start is not None:\n        groups.append((start, len(is_null) - 1))\n    \n    return groups\n\ndef get_last_valid_before(series: pd.Series, index: int) -> Optional[float]:\n    \"\"\"指定インデックス前の最後の有効値を取得\"\"\"\n    if index == 0:\n        return None\n    \n    for i in range(index - 1, -1, -1):\n        if not pd.isna(series.iloc[i]):\n            return series.iloc[i]\n    return None\n\ndef get_first_valid_after(series: pd.Series, index: int) -> Optional[float]:\n    \"\"\"指定インデックス後の最初の有効値を取得\"\"\"\n    if index == len(series) - 1:\n        return None\n    \n    for i in range(index + 1, len(series)):\n        if not pd.isna(series.iloc[i]):\n            return series.iloc[i]\n    return None\n\ndef linear_interpolate(start_val: float, end_val: float, num_points: int) -> np.ndarray:\n    \"\"\"\n    線形補間による値の生成\n    \n    Parameters:\n    -----------\n    start_val : float\n        開始値\n    end_val : float\n        終了値\n    num_points : int\n        補間する点数\n    \n    Returns:\n    --------\n    np.ndarray\n        補間された値の配列\n    \"\"\"\n    if num_points == 1:\n        return np.array([(start_val + end_val) / 2], dtype=np.float32)\n    \n    # 線形補間：y = start_val + (end_val - start_val) * t\n    # t は 1/(num_points+1) から num_points/(num_points+1) まで\n    t_values = np.linspace(1, num_points, num_points) / (num_points + 1)\n    interpolated = np.float32(start_val + (end_val - start_val) * t_values)\n    \n    return interpolated","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:27.975408Z","iopub.execute_input":"2025-10-19T16:37:27.975882Z","iopub.status.idle":"2025-10-19T16:37:28.002689Z","shell.execute_reply.started":"2025-10-19T16:37:27.975852Z","shell.execute_reply":"2025-10-19T16:37:28.001059Z"},"jupyter":{"source_hidden":true},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TrackingDataset:\n    def __init__(self, lab_id, video_id, phase, pix_per_cm_approx, tracking_df, annotation_df, behaviors_labeled):\n        self.lab_id = lab_id\n        self.video_id = video_id\n        self.phase = phase\n        self.pix_per_cm_approx = pix_per_cm_approx\n        self.tracking_df = tracking_df\n        self.annotaion_df = annotation_df\n        self.behaviors_labeled = behaviors_labeled\n\n    def get_active_action(self, agent_id, target_id):\n        if agent_id == target_id:\n            at_key = f\"mouse{agent_id},self\"\n        else:    \n            at_key = f\"mouse{agent_id},mouse{target_id}\"\n\n        active_actions = []\n        for action in self.behaviors_labeled:\n            if at_key in action:\n                active_actions.append(action.split(\",\")[-1])\n        return active_actions        \n\n    @classmethod\n    def from_row(cls, row, phase, loader):\n        tracking_df = loader.read_tracking(phase, row.lab_id, row.video_id)\n        tracking_df /= row.pix_per_cm_approx\n        if phase == \"train\":\n            annotation_df = loader.read_annotation(phase, row.lab_id, row.video_id)\n        else:\n            annotation_df = None\n        behaviors_labeled = list(set(json.loads(row.behaviors_labeled)))\n        behaviors_labeled = sorted([b.replace(\"'\", \"\") for b in behaviors_labeled])\n        return cls(row.lab_id, row.video_id, phase, row.pix_per_cm_approx, tracking_df, annotation_df, behaviors_labeled)\n\n    def get_data_for_single_mouse_model(self, is_imputation):\n        if is_imputation:\n            tracking_df = interpolate_missing_values(self.tracking_df, max_gap=90)\n        else:\n            tracking_df = self.tracking_df.copy()\n\n        data_for_single_mouse_model = {}\n\n        for mouse_id in tracking_df.columns.get_level_values(0).unique().tolist():\n            single_mouse_tracking = tracking_df[mouse_id]\n\n        \n            single_mouse_tracking_w_action = single_mouse_tracking.copy()\n            single_mouse_tracking_w_action.columns = [f\"{bp}_{xy}\" for xy, bp in single_mouse_tracking_w_action.columns]\n\n            if self.annotaion_df is not None:\n                single_mouse_annotation = self.annotaion_df[(self.annotaion_df[\"agent_id\"] == mouse_id) & (self.annotaion_df[\"target_id\"] == mouse_id)]\n                current_cols = single_mouse_tracking_w_action.columns.tolist()\n                for action, start_frame, stop_frame in single_mouse_annotation[[\"action\", \"start_frame\", \"stop_frame\"]].values:\n                    if action not in current_cols:\n                        single_mouse_tracking_w_action[action] = 2\n                        current_cols.append(action)\n                \n                    single_mouse_tracking_w_action.loc[start_frame, action] = 0\n                    if start_frame < stop_frame:\n                        single_mouse_tracking_w_action.loc[start_frame+1: stop_frame+1, action] = 1\n\n            dict_key = f\"{self.lab_id}-{self.video_id}-{mouse_id}-{mouse_id}\"\n            data_for_single_mouse_model[dict_key] = {\n                \"tracking\": single_mouse_tracking_w_action,\n                \"active_action\": self.get_active_action(mouse_id, mouse_id),\n                \"video_id\": self.video_id\n            }\n        return data_for_single_mouse_model\n\n    def get_data_for_double_mouse_model(self, is_imputation):\n        if is_imputation:\n            tracking_df = interpolate_missing_values(self.tracking_df, max_gap=90)\n        else:\n            tracking_df = self.tracking_df.copy()\n\n        data_for_double_mouse_model = {}\n\n        for agent_mouse_id, target_mouse_id in itertools.permutations(tracking_df.columns.get_level_values(0).unique().tolist(), 2):\n            agent_mouse_tracking = tracking_df[agent_mouse_id]\n            agent_mouse_tracking.columns = [f\"agent_{bp}_{xy}\" for xy, bp in agent_mouse_tracking.columns]\n            target_mouse_tracking = tracking_df[target_mouse_id]\n            target_mouse_tracking.columns = [f\"target_{bp}_{xy}\" for xy, bp in target_mouse_tracking.columns]\n            double_mouse_tracking_w_action = pd.concat([agent_mouse_tracking, target_mouse_tracking], axis=1)\n\n            if self.annotaion_df is not None:\n                double_mouse_annotation = self.annotaion_df[(self.annotaion_df[\"agent_id\"] == agent_mouse_id) & (self.annotaion_df[\"target_id\"] == target_mouse_id)]\n                current_cols = double_mouse_tracking_w_action.columns.tolist()\n                for action, start_frame, stop_frame in double_mouse_annotation[[\"action\", \"start_frame\", \"stop_frame\"]].values:\n                    if action not in current_cols:\n                        double_mouse_tracking_w_action[action] = 2\n                        current_cols.append(action)\n                \n                    double_mouse_tracking_w_action.loc[start_frame, action] = 0\n                    try:\n                        if start_frame < stop_frame:\n                            double_mouse_tracking_w_action.loc[start_frame+1: stop_frame+1, action] = 1\n                    except:\n                        display(double_mouse_tracking_w_action)\n                        print(start_frame, stop_frame, self.video_id)\n            dict_key = f\"{self.lab_id}-{self.video_id}-{agent_mouse_id}-{target_mouse_id}\"\n            data_for_double_mouse_model[dict_key] = {\n                \"tracking\": double_mouse_tracking_w_action,\n                \"active_action\": self.get_active_action(agent_mouse_id, target_mouse_id),\n                \"video_id\": self.video_id,\n            }\n        return data_for_double_mouse_model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.022948Z","iopub.execute_input":"2025-10-19T16:37:28.023329Z","iopub.status.idle":"2025-10-19T16:37:28.044109Z","shell.execute_reply.started":"2025-10-19T16:37:28.023299Z","shell.execute_reply":"2025-10-19T16:37:28.042784Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_feature_and_label_cols(trackings):\n    feature_cols = []\n    target_cols = []\n\n    tracking_columns = []\n    for _, tracking in trackings.items():\n        tracking_columns += tracking[\"tracking\"].columns.tolist()\n    tracking_columns = list(set(tracking_columns))\n\n    for col in tracking_columns:\n        if (\"_x\" in col) or (\"_y\" in col):\n            feature_cols.append(col)\n        else:\n            target_cols.append(col)\n    return feature_cols, target_cols","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.046239Z","iopub.execute_input":"2025-10-19T16:37:28.046612Z","iopub.status.idle":"2025-10-19T16:37:28.072527Z","shell.execute_reply.started":"2025-10-19T16:37:28.046580Z","shell.execute_reply":"2025-10-19T16:37:28.071234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Const","metadata":{}},{"cell_type":"code","source":"SINGLE_LAB_LIST = [\n    \"AdaptableSnail\",\n    # \"BoisterousParrot\",\n    # \"CautiousGiraffe\", # Only Muli\n    # \"DeliriousFly\",\n    # \"ElegantMink\",\n    \"GroovyShrew\",\n    # \"InvincibleJellyfish\",\n    # \"JovialSwallow\",\n    \"LyricalHare\",\n    \"NiftyGoldfinch\",\n    # \"PleasantMeerkat\",\n    # \"ReflectiveManatee\",\n    # \"SparklingTapir\",\n    \"TranquilPanther\",\n    \"UppityFerret\"\n]\n\nDOUBLE_LAB_LIST = [\n    \"AdaptableSnail\",\n    \"BoisterousParrot\",\n    \"CautiousGiraffe\", # Only Muli\n    \"DeliriousFly\",\n    \"ElegantMink\",\n    \"GroovyShrew\",\n    \"InvincibleJellyfish\",\n    \"JovialSwallow\",\n    \"LyricalHare\",\n    \"NiftyGoldfinch\",\n    \"PleasantMeerkat\",\n    \"ReflectiveManatee\",\n    \"SparklingTapir\",\n    \"TranquilPanther\",\n    \"UppityFerret\"\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.074430Z","iopub.execute_input":"2025-10-19T16:37:28.074820Z","iopub.status.idle":"2025-10-19T16:37:28.093890Z","shell.execute_reply.started":"2025-10-19T16:37:28.074787Z","shell.execute_reply":"2025-10-19T16:37:28.092626Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SEQ_LEN = 90\n# WINDOWS = 20\n# TRAIN_RATE = 0.5\n# LAB_ID = SINGLE_LAB_LIST[5]\n# PHASE = \"train\"\n# ACTION_TYPE = \"double\"\n# if ACTION_TYPE == \"single\":\n#     LAB_LIST = SINGLE_LAB_LIST\n# elif ACTION_TYPE == \"double\":\n#     LAB_LIST = DOUBLE_LAB_LIST\n# LAB_ID = LAB_LIST[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.095037Z","iopub.execute_input":"2025-10-19T16:37:28.095570Z","iopub.status.idle":"2025-10-19T16:37:28.118859Z","shell.execute_reply.started":"2025-10-19T16:37:28.095535Z","shell.execute_reply":"2025-10-19T16:37:28.117595Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Setup Dataset","metadata":{}},{"cell_type":"code","source":"def load_tracking_dataset(lab_id, phase, action_type, meta_df):\n    trackings = {}\n    for _, row in tqdm(meta_df[meta_df[\"lab_id\"] == lab_id].iterrows()):\n        if \"MABe22\" in row.lab_id: continue\n        try:\n            tracking_dataset = TrackingDataset.from_row(row, phase, loader)\n            features_col = tracking_dataset.tracking_df.columns\n            if action_type == \"single\":\n                trackings |= tracking_dataset.get_data_for_single_mouse_model(is_imputation=True)\n            elif action_type == \"double\":\n                trackings |= tracking_dataset.get_data_for_double_mouse_model(is_imputation=True)\n        except FileNotFoundError as e:\n            print(e)\n    return trackings","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.120954Z","iopub.execute_input":"2025-10-19T16:37:28.121386Z","iopub.status.idle":"2025-10-19T16:37:28.139629Z","shell.execute_reply.started":"2025-10-19T16:37:28.121354Z","shell.execute_reply":"2025-10-19T16:37:28.138562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def standarization_columns(trackings, feature_cols, target_cols):\n    standarized_trackings = []\n    for _, tracking_dict in trackings.items():\n        tracking = tracking_dict[\"tracking\"]\n        # 説明変数のカラム数が足りないデータに対し、欠損列を追加\n        for f_col in feature_cols:\n            if f_col not in tracking.columns:\n                tracking[f_col] = np.nan\n    \n        if len(set(tracking.columns) & set(target_cols)) and (len(target_cols) > 0) == 0:\n            continue\n    \n        # 目的変数のカラムが足りないデータに対し、非アクション値を追加\n        for t_col in target_cols:\n            if t_col not in tracking.columns:\n                tracking[t_col] = -1\n    \n        fixed_columns = feature_cols + target_cols\n        standarized_trackings.append(tracking[fixed_columns])\n    return standarized_trackings\n\ndef get_max_cm(trackings, feature_cols):\n    max_cm = 0\n    for tracking in trackings:\n        _max_cm = tracking[feature_cols].fillna(0).max().max()\n        if max_cm < _max_cm:\n            max_cm = _max_cm\n    return max_cm\n\ndef standarization_tracks_vals(trackings, feature_cols, max_cm=None):\n    if max_cm is None:\n        max_cm = get_max_cm(trackings, feature_cols)\n    standarized_trackings = []\n    for tracking in trackings:\n        tracking[feature_cols] /= max_cm\n        standarized_trackings.append(tracking)\n    return standarized_trackings, max_cm\n\ndef train_test_split(trackings, train_rate):\n    tracking_train_list = []\n    tracking_valid_list = []\n    \n    for tracking in trackings:\n        L = len(tracking)\n        tracking_train_list.append(tracking[tracking.index < L * train_rate].values)\n        tracking_valid_list.append(tracking[tracking.index >= L * train_rate].values)\n\n    return tracking_train_list, tracking_valid_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.189530Z","iopub.execute_input":"2025-10-19T16:37:28.189856Z","iopub.status.idle":"2025-10-19T16:37:28.200573Z","shell.execute_reply.started":"2025-10-19T16:37:28.189832Z","shell.execute_reply":"2025-10-19T16:37:28.199377Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_weight_torch(trackings, target_cols):\n    weights = {k: {0: 0, 1: 0, 2: 0, \"total\": 0} for k in target_cols}\n    for tracking in trackings:\n        for k in target_cols:\n            count = Counter(tracking[k].tolist())\n            for v, count in count.items():\n                if v == -1:\n                    continue\n                weights[k][v] += count\n                weights[k][\"total\"] += count\n\n    torch_weight = torch.ones((len(target_cols), 3))\n    for ki, k in enumerate(target_cols):\n        action_torch_weights = torch.ones(3)\n        for v in range(3):\n            action_torch_weights[v] = weights[k][\"total\"] / weights[k][v]\n        torch_weight[ki] = action_torch_weights\n    return torch_weight\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.245731Z","iopub.execute_input":"2025-10-19T16:37:28.246066Z","iopub.status.idle":"2025-10-19T16:37:28.253943Z","shell.execute_reply.started":"2025-10-19T16:37:28.246044Z","shell.execute_reply":"2025-10-19T16:37:28.252845Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"code","source":"class MouseBehaviorDataset(Dataset):\n    def __init__(self, data_list: List[np.ndarray], sequence_length: int = 150, n_cols=1, windows=1):\n        \"\"\"\n        data_list: List of arrays, each with shape (N_k, C+target_dims)\n        sequence_length: Window size for training\n        \"\"\"\n        self.sequence_length = sequence_length\n        self.sequences = []\n        self.targets = []\n        \n        for mouse_data in data_list:\n            is_end = False\n            # Assume last columns are multi-label targets\n            features = mouse_data[:, :-n_cols]  # C features (with missing values)\n            targets = mouse_data[:, -n_cols:]   # 3 actions (a, b, c)\n            \n            # Create sliding windows\n            step = math.ceil((len(mouse_data) - sequence_length) / windows) + 1\n            for i in range(step):\n                seq_features = features[i*windows:i*windows+sequence_length]\n                seq_targets = targets[i*windows:i*windows+sequence_length]\n                if len(seq_features) < sequence_length:\n                    seq_features = features[-sequence_length:]\n                    seq_targets = targets[-sequence_length:]\n                # if len(seq_features) < sequence_length:\n                #     seq_features = features[-sequence_length:]\n                #     seq_targets = targets[-sequence_length:]\n                #     is_end = True\n                \n                self.sequences.append(seq_features.astype(np.float32))\n                self.targets.append(seq_targets.astype(np.int64))\n                # if is_end:\n                #     break\n    \n    def __len__(self):\n        return len(self.sequences)\n    \n    def __getitem__(self, idx):\n        return torch.from_numpy(self.sequences[idx]), torch.from_numpy(self.targets[idx])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.328719Z","iopub.execute_input":"2025-10-19T16:37:28.329066Z","iopub.status.idle":"2025-10-19T16:37:28.339441Z","shell.execute_reply.started":"2025-10-19T16:37:28.329042Z","shell.execute_reply":"2025-10-19T16:37:28.338150Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MouseBehaviorTestDataset(Dataset):\n    def __init__(self, data_list: List[np.ndarray], sequence_length: int = 150, windows=1):\n        \"\"\"\n        data_list: List of arrays, each with shape (N_k, C+target_dims)\n        sequence_length: Window size for training\n        \"\"\"\n        self.sequence_length = sequence_length\n        self.sequences = []\n        self.targets = []\n        \n        for mouse_data in data_list:\n            is_end = False\n            # Assume last columns are multi-label targets\n            \n            # Create sliding windows\n            step = math.ceil((len(mouse_data) - sequence_length) / windows) + 1\n            for i in range(step):\n                seq_features = mouse_data[i*windows:i*windows+sequence_length]\n                if len(seq_features) < sequence_length:\n                    seq_features = mouse_data[-sequence_length:]\n                \n                self.sequences.append(seq_features.astype(np.float32))\n                # if is_end:\n                #     break\n    \n    def __len__(self):\n        return len(self.sequences)\n    \n    def __getitem__(self, idx):\n        return torch.from_numpy(self.sequences[idx])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.340935Z","iopub.execute_input":"2025-10-19T16:37:28.341421Z","iopub.status.idle":"2025-10-19T16:37:28.368050Z","shell.execute_reply.started":"2025-10-19T16:37:28.341378Z","shell.execute_reply":"2025-10-19T16:37:28.366621Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MaskedBiLSTM(nn.Module):\n    def __init__(self, \n                 input_dim: int, spatial_hidden_dim=32, \n                 lstm_input_dim=32, lstm_hidden_dim: int = 32, \n                 num_actions: int = 1, num_layers: int = 4):\n        super().__init__()\n        # self.hidden_dim = hidden_dim\n        self.num_actions = num_actions\n\n        self.spatial_encoder = nn.Sequential(\n            nn.Linear(input_dim, spatial_hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.1),\n            nn.Linear(spatial_hidden_dim, lstm_input_dim),\n            nn.ReLU()\n        )\n        \n        # BiLSTM layers\n        self.lstm = nn.LSTM(lstm_input_dim, lstm_hidden_dim, num_layers, \n                           batch_first=True, bidirectional=True, dropout=0.1)\n        \n        # Action classifiers (3 actions: a, b, c)\n        self.action_classifiers = nn.ModuleList([\n            nn.Linear(lstm_hidden_dim * 2, 3) for _ in range(num_actions)  # 3 classes: B, I, O\n        ])\n\n        self.layer_norm = nn.LayerNorm(lstm_hidden_dim*2)\n        self.relu = nn.ReLU()\n        \n        self.dropout = nn.Dropout(0.1)\n\n    def _init_weight(self):\n        for name, param in self.lstm.named_parameters():\n            if \"weight_ih\" in name:\n                nn.init.xavier_uniform_(param.data)\n\n            elif \"weight_hh\" in name:\n                nn.init.orthognal_(param.data)\n\n            elif \"bias\" in name:\n                param.data.fill_(0)\n                n = param.size(9)\n                param.data[n//4:n//2].fill(1)\n        for classifier in self.action_classifiers:\n            nn.init.xavier_uniform_(classifier.weight)\n            nn.init.zeros_(classifier.bias)\n        \n    def forward(self, x, mask=None):\n        batch_size, seq_len, input_dim = x.shape\n        \n        # Create mask for missing values (assume NaN represents missing)\n        if mask is None:\n            mask = ~torch.isnan(x).any(dim=-1)  # (batch, seq_len)\n        \n        # Replace NaN with zeros\n        x = torch.nan_to_num(x, -1.0)\n        x = self.spatial_encoder(x)\n        \n        # LSTM forward pass\n        lstm_out, _ = self.lstm(x)  # (batch, seq_len, hidden_dim*2)\n        lstm_out = self.relu(lstm_out)\n        lstm_out = self.layer_norm(lstm_out)\n        lstm_out = self.dropout(lstm_out)\n        \n        # # Apply mask to LSTM output\n        # if mask is not None:\n        #     mask_expanded = mask.unsqueeze(-1).expand_as(lstm_out)\n        #     lstm_out = lstm_out * mask_expanded.float()\n        \n        # Multi-label classification for each action\n        action_logits = []\n        for classifier in self.action_classifiers:\n            logits = classifier(lstm_out)  # (batch, seq_len, 3)\n            action_logits.append(logits)\n        \n        return torch.stack(action_logits, dim=-2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.370150Z","iopub.execute_input":"2025-10-19T16:37:28.370473Z","iopub.status.idle":"2025-10-19T16:37:28.395458Z","shell.execute_reply.started":"2025-10-19T16:37:28.370447Z","shell.execute_reply":"2025-10-19T16:37:28.394405Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(train_data: List[np.ndarray], \n                val_data: List[np.ndarray],\n                torch_weight,\n                input_dim: int = None,\n                sequence_length: int = 150,\n                epochs: int = 2,\n                batch_size: int = 512,\n                learning_rate: float = 0.001, n_cols=1, windows=1, suffix=\"\"):\n    \n    # Create datasets\n    train_dataset = MouseBehaviorDataset(train_data, sequence_length, n_cols, windows)\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, pin_memory=True,      # GPU転送を高速化\n    num_workers=4)\n    \n    val_dataset = MouseBehaviorDataset(val_data, sequence_length, n_cols, windows)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, pin_memory=True,      # GPU転送を高速化\n    num_workers=4)\n\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = MaskedBiLSTM(input_dim, num_actions=n_cols)\n    model.to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate, weight_decay=0.01, betas=(0.9, 0.999), eps=1e-8)\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n        optimizer,\n        mode=\"min\",\n        factor=0.5,\n        patience=5,\n        verbose=True,\n        min_lr=1e-6\n    )\n      # Ignore masked positions\n\n    patience = 10\n    minibatch_count = 0\n    best_val_acc = -np.inf\n    \n    # Training loop\n    for epoch in tqdm(range(epochs)):\n        model.train()\n        total_loss = 0\n        \n        for batch_features, batch_targets in train_loader:\n            batch_features = batch_features.to(device)\n            batch_targets = batch_targets.to(device)\n            \n            optimizer.zero_grad()\n            \n            # Forward pass\n            predictions = model(batch_features)  # (batch, seq, num_actions, 3)\n            \n            # Calculate loss for each action\n            loss = 0\n            for action_idx in range(n_cols):\n                action_pred = predictions[:, :, action_idx]  # (batch, seq, 3)\n                action_target = batch_targets[:, :, action_idx]  # (batch, seq)\n                criterion = nn.CrossEntropyLoss(ignore_index=-1, weight=torch_weight[action_idx].to(device))\n                _loss = criterion(action_pred.reshape(-1, 3), action_target.reshape(-1))\n                loss += _loss\n            \n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            total_loss += loss.item()\n        print(f'Epoch {epoch}, Loss: {total_loss/len(train_loader):.4f}')\n        \n        if val_data:\n            tr_f1, _ = evaluate_model(model, train_loader, device, n_cols)\n            print(f'Train F1: {tr_f1:.4f}') \n            val_f1, val_loss = evaluate_model(model, val_loader, device, n_cols)\n            print(f'Validation F1: {val_f1:.4f}') \n            # torch.save(model.state_dict(), f\"model_{epoch}_{suffix}.pth\")\n            scheduler.step(val_loss)\n    torch.save(model.state_dict(), f\"model_{suffix}.pth\")\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.396652Z","iopub.execute_input":"2025-10-19T16:37:28.396948Z","iopub.status.idle":"2025-10-19T16:37:28.428526Z","shell.execute_reply.started":"2025-10-19T16:37:28.396927Z","shell.execute_reply":"2025-10-19T16:37:28.427237Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(model, test_data: List[np.ndarray], sequence_length: int = 150, batch_size: int = 32, windows=1):\n    \"\"\"Inference function\"\"\"\n    test_dataset = MouseBehaviorTestDataset(test_data, sequence_length, windows)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, pin_memory=True,      # GPU転送を高速化\n    num_workers=4)\n    \n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model.to(device)\n    model.eval()\n    \n    all_predictions = []\n    \n    with torch.no_grad():\n        for batch_features in test_loader:\n            batch_features = batch_features.to(device)\n            \n            # Forward pass\n            bio_predictions = model(batch_features)  # (batch, seq, num_actions, 3)\n            bio_predictions = F.softmax(bio_predictions, dim=-1)\n            \n            # Convert to multi-label predictions\n            # multilabel_pred = bio_to_multilabel(bio_predictions)\n            all_predictions.append(bio_predictions.cpu().numpy())\n    \n    return np.concatenate(all_predictions, axis=0)\n\ndef evaluate_model(model, data_loader, device, n_cols):\n    \"\"\"Evaluation function\"\"\"\n    model.eval()\n    action_f1s = {i: {\"tps\": 0, \"fps\": 0, \"fns\": 0} for i in range(n_cols)}\n    beta = 1\n    val_loss = 0\n    \n    with torch.no_grad():\n        for batch_features, batch_targets in data_loader:\n            batch_features = batch_features.to(device)\n            batch_targets = batch_targets.to(device)\n            \n            predictions = model(batch_features)\n            bio_pred = predictions.argmax(dim=-1)\n\n            for i in range(n_cols):\n                targets_flat = batch_targets[:, :, i].flatten()\n                pred_flat = bio_pred[:, :, i].flatten()\n\n                criterion = nn.CrossEntropyLoss(ignore_index=-1, weight=torch_weight[i].to(device))\n                _loss = criterion(predictions[:, :, i].reshape(-1, 3), targets_flat.reshape(-1))\n                val_loss += _loss / len(data_loader)\n    \n                target_active = (targets_flat == 0) | (targets_flat == 1)\n                target_inactive = (targets_flat == 2)\n                pred_active = (pred_flat == 0) | (pred_flat == 1)\n                pred_inactive = (pred_flat == 2)\n                action_f1s[i][\"tps\"] += (target_active & pred_active).sum().item()\n                action_f1s[i][\"fns\"] += (target_active & pred_inactive).sum().item()\n                action_f1s[i][\"fps\"] += (target_inactive & pred_active).sum().item()\n\n    fscore = 0\n    for i in range(n_cols):\n        try:\n            fbeta = (1 + beta**2) * action_f1s[i][\"tps\"] / ((1 + beta**2) * action_f1s[i][\"tps\"] + beta**2 * action_f1s[i][\"fns\"] + action_f1s[i][\"fps\"])\n        except:\n            fbeta = 0\n        fscore += fbeta / n_cols\n    return fscore, val_loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.430604Z","iopub.execute_input":"2025-10-19T16:37:28.431016Z","iopub.status.idle":"2025-10-19T16:37:28.457681Z","shell.execute_reply.started":"2025-10-19T16:37:28.430983Z","shell.execute_reply":"2025-10-19T16:37:28.456395Z"},"_kg_hide-input":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def fixed_pred_per_frame(frame_len, prediction, target_cols, seq_len, windows):\n    final_prob = np.zeros((frame_len, len(target_cols), 3))\n    step = math.ceil((frame_len - seq_len) // windows) + 1\n    p_index = 0\n    for i in range(step):\n        if len(final_prob[i*WINDOWS:i*WINDOWS+SEQ_LEN, :, :]) == SEQ_LEN:\n            final_prob[i*WINDOWS:i*WINDOWS+SEQ_LEN, :, :] += prediction[p_index]\n        else:\n            final_prob[-SEQ_LEN:, :, :] += prediction[p_index]\n        p_index += 1\n    final_prob = final_prob / final_prob.sum(axis=-1, keepdims=True)\n    return final_prob","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.459200Z","iopub.execute_input":"2025-10-19T16:37:28.459578Z","iopub.status.idle":"2025-10-19T16:37:28.481230Z","shell.execute_reply.started":"2025-10-19T16:37:28.459554Z","shell.execute_reply":"2025-10-19T16:37:28.479985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Robustify function\ndef robustify(submission, meta_df):\n\n    old_submission = submission.copy()\n    submission = submission[submission.start_frame < submission.stop_frame]\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped frames with start >= stop\")\n    \n    old_submission = submission.copy()\n    group_list = []\n    for _, group in submission.groupby(['video_id', 'agent_id', 'target_id']):\n        group = group.sort_values('start_frame').reset_index(drop=True)\n        mask = np.ones(len(group), dtype=bool)\n        last_stop_frame = 0\n        non_fixed = True\n        while non_fixed:\n            last_stop_frame = 0\n            for i, (_, row) in enumerate(group.iterrows()):\n                if row['start_frame'] < last_stop_frame:\n                    if row['stop_frame'] < last_stop_frame:\n                        mask[i] = False\n                    else:\n                        group.loc[i, \"start_frame\"] = last_stop_frame\n                        break\n                else:\n                    last_stop_frame = row['stop_frame']\n            else:\n                non_fixed = False\n        group_list.append(group[mask])\n    submission = pd.concat(group_list)\n    if len(submission) != len(old_submission):\n        print(\"ERROR: Dropped duplicate frames\")\n\n    submission = submission.reset_index(drop=True)\n    return submission","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.482327Z","iopub.execute_input":"2025-10-19T16:37:28.482600Z","iopub.status.idle":"2025-10-19T16:37:28.507724Z","shell.execute_reply.started":"2025-10-19T16:37:28.482579Z","shell.execute_reply":"2025-10-19T16:37:28.506443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SEQ_LEN = 90\n# TRAIN_RATE = 0.5\n# PHASE = \"train\"\n# thres_info = {}\n# label_name_info = {}\n\n# root_dir = Path(\"/kaggle/input/MABe-mouse-behavior-detection\")\n# loader = RawTableLoader(root_dir)\n# train_df = loader.read_train()\n\n# for action_type in [\"single\", \"double\"]:\n#     if action_type == \"single\":\n#         LAB_LIST = SINGLE_LAB_LIST\n#         WINDOWS = 1\n#     elif action_type == \"double\":\n#         LAB_LIST = DOUBLE_LAB_LIST\n#         WINDOWS = 20\n\n#     for lab_id in LAB_LIST:\n#         trackings = load_tracking_dataset(lab_id=lab_id, phase=PHASE, action_type=action_type, meta_df=train_df)\n#         feature_cols, target_cols = get_feature_and_label_cols(trackings)\n        \n#         trackings_for_modeling = standarization_columns(trackings, feature_cols, target_cols)\n#         trackings_for_modeling, max_cp = standarization_tracks_vals(trackings_for_modeling, feature_cols)\n#         label_name_info[f\"{lab_id}_{action_type}\"] = {\n#             \"feature_cols\": feature_cols,\n#             \"label_cols\": target_cols,\n#             \"max_cp\": max_cp\n#         }\n#         train_tracking, valid_tracking = train_test_split(trackings_for_modeling, train_rate=TRAIN_RATE)\n#         torch_weight = get_weight_torch(trackings_for_modeling, target_cols)\n    \n#         model = train_model(\n#             train_tracking, \n#             valid_tracking, \n#             torch_weight=torch_weight,\n#             input_dim=len(feature_cols),\n#             sequence_length=SEQ_LEN, \n#             epochs=10, \n#             batch_size=256, \n#             learning_rate=0.001,\n#             n_cols=len(target_cols),\n#             windows=WINDOWS,\n#             suffix = f\"{action_type}_{lab_id}\",\n#         )\n\n#         scores = []\n#         for valid_tracking_val in valid_tracking:\n#             prediction = predict(\n#                 model, \n#                 [valid_tracking_val[:, :-len(target_cols)]], \n#                 sequence_length=SEQ_LEN, \n#                 batch_size=1024, \n#                 n_cols=len(target_cols),\n#                 windows=WINDOWS\n#             )\n        \n#             pred_per_frame = fixed_pred_per_frame(len(valid_tracking_val), prediction, target_cols, SEQ_LEN, WINDOWS)\n        \n#             score_info = {}\n#             for i, target_col in enumerate(target_cols):\n#                 score_info[f\"{target_col}_pred\"] = 1 - pred_per_frame[:, i, 2]\n#                 score_info[f\"{target_col}_correct\"] = valid_tracking_val[:, -len(target_cols) + i]\n                \n#             score = pd.DataFrame(score_info)\n#             scores.append(score)\n#         scores = pd.concat(scores)\n        \n#         fbeta = 0\n#         thres_info_per_lab = {}\n#         for target_col in target_cols:\n#             best_fbeta_per_action = 0\n#             best_fbeta_thres_per_action = -1\n#             for thres in np.arange(0, 1, 0.01):\n#                 tps = len(scores[(scores[f\"{target_col}_pred\"] > thres) & scores[f\"{target_col}_correct\"].isin([0,1])])\n#                 fps = len(scores[(scores[f\"{target_col}_pred\"] > thres) & scores[f\"{target_col}_correct\"].isin([2])])\n#                 fns = len(scores[(scores[f\"{target_col}_pred\"] <= thres) & scores[f\"{target_col}_correct\"].isin([0,1])])\n#                 beta = 1\n#                 _fbeta = (1 + beta**2) * tps / ((1 + beta**2) * tps + beta**2 * fns + fps)\n#                 if best_fbeta_per_action < _fbeta:\n#                     best_fbeta_thres_per_action = thres\n#                     best_fbeta_per_action = _fbeta\n#             thres_info_per_lab[target_col] = float(best_fbeta_thres_per_action)\n#             fbeta += best_fbeta_per_action / len(target_cols)\n#             print(f\"action: {target_col}, thres: {best_fbeta_thres_per_action}, fbeta: {best_fbeta_per_action}\")\n#         thres_info[f\"{lab_id}_{action_type}\"] = thres_info_per_lab\n#         print(f\"fbeta: {fbeta}\")\n\n# with open('config.yaml', 'w') as f:\n#     yaml.dump(label_name_info, f, default_flow_style=False, sort_keys=False)\n\n# with open('thres_info.yaml', 'w') as f:\n#     yaml.dump(thres_info, f, default_flow_style=False, sort_keys=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-19T16:37:28.510191Z","iopub.execute_input":"2025-10-19T16:37:28.510549Z","execution_failed":"2025-10-19T16:43:34.141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEQ_LEN = 90\nMODEL_DIR = \"/kaggle/input/mabe-lstm-models\"\nTRAIN_RATE = 0.5\nPHASE = \"test\"\n\nroot_dir = Path(\"/kaggle/input/MABe-mouse-behavior-detection\")\nloader = RawTableLoader(root_dir)\ntest_df = loader.read_test()\n\nwith open(os.path.join(MODEL_DIR, 'config.yaml'), 'r', encoding='utf-8') as f:\n    label_name_info = yaml.safe_load(f)\n\nwith open(os.path.join(MODEL_DIR, 'thres_info.yaml'), 'r', encoding='utf-8') as f:\n    thres_info = yaml.safe_load(f)\n\npredictions = []\n\nfor action_type in [\"single\", \"double\"]:\n    if action_type == \"single\":\n        LAB_LIST = SINGLE_LAB_LIST\n        WINDOWS = 1\n    elif action_type == \"double\":\n        LAB_LIST = DOUBLE_LAB_LIST\n        WINDOWS = 20\n\n    for lab_id in LAB_LIST:\n        trackings = load_tracking_dataset(lab_id=lab_id, phase=PHASE, action_type=action_type, meta_df=test_df)\n        feature_cols = label_name_info[f\"{lab_id}_{action_type}\"][\"feature_cols\"]\n        target_cols = label_name_info[f\"{lab_id}_{action_type}\"][\"label_cols\"]\n        max_cm = label_name_info[f\"{lab_id}_{action_type}\"][\"max_cp\"]\n        thres_info_per_lab = thres_info[f\"{lab_id}_{action_type}\"]\n\n        \n        trackings_for_modeling = standarization_columns(trackings, feature_cols, [])\n        trackings_for_modeling, _ = standarization_tracks_vals(trackings_for_modeling, feature_cols, max_cm=max_cm)\n\n        if len(trackings_for_modeling) == 0:\n            continue\n        trackings_for_modeling = [tracking_for_modeling.values for tracking_for_modeling in trackings_for_modeling]\n        input_dim = trackings_for_modeling[0].shape[1]\n\n        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        suffix = f\"{action_type}_{lab_id}\"\n        state_dict = torch.load(os.path.join(MODEL_DIR, f\"model_{suffix}.pth\"), map_location='cpu')\n        model = MaskedBiLSTM(input_dim, num_actions=len(target_cols))\n        model.load_state_dict(state_dict)\n        model.to(device)\n        \n        scores = []\n        for tracking_for_modeling, (keys, tracking_info) in zip(trackings_for_modeling, trackings.items()):\n            _, video_id, agent_id, target_id = keys.split(\"-\")\n            prediction = predict(\n                model, \n                [tracking_for_modeling], \n                sequence_length=SEQ_LEN, \n                batch_size=1024, \n                windows=WINDOWS\n            )\n\n            pred_per_frame = fixed_pred_per_frame(len(tracking_for_modeling), prediction, target_cols, SEQ_LEN, WINDOWS)\n            active_action = tracking_info[\"active_action\"]\n\n            assert len(set(target_cols) | set(active_action)) == len(target_cols)\n            for i, target_col in enumerate(target_cols):\n                if target_col not in active_action:\n                    continue\n                score_info = {}\n                score_info[\"video_frame\"] = tracking_info[\"tracking\"].index\n                score_info[\"action\"] = target_col\n                score_info[\"pred\"] = ((1 - pred_per_frame[:, i, 2]) > thres_info_per_lab[target_col]) * 1\n                score = pd.DataFrame(score_info)\n\n                score = score[score[\"pred\"] == 1]\n                score = score.sort_values(\"video_frame\")\n                score['frame_diff'] = score[\"video_frame\"].diff()\n                \n                # 新しいセグメントの開始点を特定（差分が1より大きい場合）\n                score['new_segment'] = (score['frame_diff'] > 1) | (score['frame_diff'].isna())\n                score['segment_id'] = score['new_segment'].cumsum()\n                \n                # セグメントごとに開始・終了フレームを取得\n                for segment_id in score['segment_id'].unique():\n                    segment_frames = score[score['segment_id'] == segment_id][\"video_frame\"]\n                    start_frame = segment_frames.min()\n                    stop_frame = segment_frames.max()\n                    \n                    predictions.append({\n                        \"video_id\": video_id,\n                        \"agent_id\": f\"mouse{agent_id}\",\n                        \"target_id\": f\"mouse{target_id}\" if agent_id != target_id else \"self\",\n                        'action': target_col,\n                        'start_frame': start_frame,\n                        'stop_frame': stop_frame\n                    })\n            \n                        \n\npredictions = pd.DataFrame(predictions).reset_index(drop=True)\nrobustify_predictions = robustify(predictions, test_df)\nrobustify_predictions.index.name = 'row_id'\n\nif len(prediction) == 0:\n    predictions = pd.DataFrame(\n        dict(\n            video_id=438887472,\n            agent_id='mouse1',\n            target_id='self',\n            action='rear',\n            start_frame='278',\n            stop_frame='500'\n        ), index=[44]\n    )\nrobustify_predictions.to_csv('submission.csv')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-19T16:43:34.142Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# import polars as pl\n# class HostVisibleError(Exception):\n#     pass\n\n# def single_lab_f1(lab_solution: pl.DataFrame, lab_submission: pl.DataFrame, beta: float = 1) -> float:\n#     label_frames: defaultdict[str, set[int]] = defaultdict(set)\n#     prediction_frames: defaultdict[str, set[int]] = defaultdict(set)\n\n#     for row in lab_solution.to_dicts():\n#         label_frames[row['label_key']].update(range(row['start_frame'], row['stop_frame']))\n\n#     for video in lab_solution['video_id'].unique():\n#         active_labels: str = lab_solution.filter(pl.col('video_id') == video)['behaviors_labeled'].first()\n#         active_labels: set[str] = set(json.loads(active_labels))\n#         predicted_mouse_pairs: defaultdict[str, set[int]] = defaultdict(set)\n\n#         for row in lab_submission.filter(pl.col('video_id') == video).to_dicts():\n#             if ','.join([str(row['agent_id']), str(row['target_id']), row['action']]) not in active_labels:\n#                 continue\n           \n#             new_frames = set(range(row['start_frame'], row['stop_frame']))\n#             new_frames = new_frames.difference(prediction_frames[row['prediction_key']])\n#             prediction_pair = ','.join([str(row['agent_id']), str(row['target_id'])])\n#             if predicted_mouse_pairs[prediction_pair].intersection(new_frames):\n#                 print(prediction_pair)\n#                 print(predicted_mouse_pairs[prediction_pair])\n#                 print(new_frames)\n#                 raise HostVisibleError('Multiple predictions for the same frame from one agent/target pair')\n#             prediction_frames[row['prediction_key']].update(new_frames)\n#             predicted_mouse_pairs[prediction_pair].update(new_frames)\n\n#     tps = defaultdict(int)\n#     fns = defaultdict(int)\n#     fps = defaultdict(int)\n#     for key, pred_frames in prediction_frames.items():\n#         action = key.split('_')[-1]\n#         matched_label_frames = label_frames[key]\n#         tps[action] += len(pred_frames.intersection(matched_label_frames))\n#         fns[action] += len(matched_label_frames.difference(pred_frames))\n#         fps[action] += len(pred_frames.difference(matched_label_frames))\n\n#     distinct_actions = set()\n#     for key, frames in label_frames.items():\n#         action = key.split('_')[-1]\n#         distinct_actions.add(action)\n#         if key not in prediction_frames:\n#             fns[action] += len(frames)\n\n#     action_f1s = []\n#     for action in distinct_actions:\n#         if tps[action] + fns[action] + fps[action] == 0:\n#             action_f1s.append(0)\n#         else:\n#             action_f1s.append((1 + beta**2) * tps[action] / ((1 + beta**2) * tps[action] + beta**2 * fns[action] + fps[action]))\n#     return sum(action_f1s) / len(action_f1s)\n\n# def mouse_fbeta(solution: pd.DataFrame, submission: pd.DataFrame, beta: float = 1) -> float:\n#     if len(solution) == 0 or len(submission) == 0:\n#         raise ValueError('Missing solution or submission data')\n\n#     expected_cols = ['video_id', 'agent_id', 'target_id', 'action', 'start_frame', 'stop_frame']\n\n#     for col in expected_cols:\n#         if col not in solution.columns:\n#             raise ValueError(f'Solution is missing column {col}')\n#         if col not in submission.columns:\n#             raise ValueError(f'Submission is missing column {col}')\n\n#     solution: pl.DataFrame = pl.DataFrame(solution)\n#     submission: pl.DataFrame = pl.DataFrame(submission)\n#     assert (solution['start_frame'] <= solution['stop_frame']).all()\n#     assert (submission['start_frame'] <= submission['stop_frame']).all()\n#     solution_videos = set(solution['video_id'].unique())\n#     submission = submission.filter(pl.col('video_id').is_in(solution_videos))\n\n#     solution = solution.with_columns(\n#         pl.concat_str(\n#             [\n#                 pl.col('video_id').cast(pl.Utf8),\n#                 pl.col('agent_id').cast(pl.Utf8),\n#                 pl.col('target_id').cast(pl.Utf8),\n#                 pl.col('action'),\n#             ],\n#             separator='_',\n#         ).alias('label_key'),\n#     )\n#     submission = submission.with_columns(\n#         pl.concat_str(\n#             [\n#                 pl.col('video_id').cast(pl.Utf8),\n#                 pl.col('agent_id').cast(pl.Utf8),\n#                 pl.col('target_id').cast(pl.Utf8),\n#                 pl.col('action'),\n#             ],\n#             separator='_',\n#         ).alias('prediction_key'),\n#     )\n\n#     lab_scores = []\n#     for lab in solution['lab_id'].unique():\n#         lab_solution = solution.filter(pl.col('lab_id') == lab).clone()\n#         lab_videos = set(lab_solution['video_id'].unique())\n#         lab_submission = submission.filter(pl.col('video_id').is_in(lab_videos)).clone()\n#         lab_scores.append(single_lab_f1(lab_solution, lab_submission, beta=beta))\n\n#     return sum(lab_scores) / len(lab_scores)\n\n# def score(solution: pd.DataFrame, submission: pd.DataFrame, row_id_column_name: str, beta: float = 1) -> float:\n#     solution = solution.drop(row_id_column_name, axis='columns', errors='ignore')\n#     submission = submission.drop(row_id_column_name, axis='columns', errors='ignore')\n#     return mouse_fbeta(solution, submission, beta=beta)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-10-19T16:43:34.142Z"}},"outputs":[],"execution_count":null}]}