{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":13149332,"sourceType":"datasetVersion","datasetId":8270626}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport warnings\nimport math\nimport json\nimport random\nfrom copy import deepcopy\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nimport joblib\nimport polars as pl\nfrom tqdm.auto import tqdm\n\n# Scikit-learn\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.model_selection import StratifiedGroupKFold, PredefinedSplit\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import f1_score\n\n# SciPy\nfrom scipy.spatial.transform import Rotation as R\n\n# PyTorch\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nwarnings.filterwarnings(\"ignore\")\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")\n\n# =============================================================================\n# 1) CONFIG & SEED\n# =============================================================================\nclass Config:\n    RAW_DIR = Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")\n    IDS_DEL_PATH = Path(\"/kaggle/input/processdata/ids2del_prop_0.2.csv\")\n    EXPORT_DIR = Path(\"/kaggle/working/\")\n    BATCH_SIZE = 64\n    PAD_PERCENTILE = 94\n    LR_INIT = 3e-4\n    WD = 3e-4\n    MIXUP_ALPHA = 0.4\n    EPOCHS = 220\n    PATIENCE = 30\n    T_0 = 25\n    EMA_DECAY = 0.99\n    ACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    ROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n    SEED = 42\n\nos.makedirs(Config.EXPORT_DIR, exist_ok=True)\n\ndef set_seed(seed=42):\n    random.seed(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\nset_seed(Config.SEED)\n\n# =============================================================================\n# 2) LOSS / AUGS / UTILITIES\n# =============================================================================\nclass FocalLoss(nn.Module):\n    \"\"\"\n    Focal loss cho soft labels (mixup). Tính trên log_softmax.\n    \"\"\"\n    def __init__(self, alpha=0.25, gamma=2.0, reduction=\"mean\"):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.reduction = reduction\n\n    def forward(self, inputs, targets):\n        # inputs: logits (N, C); targets: soft labels (N, C)\n        logp = F.log_softmax(inputs, dim=1)\n        p = logp.exp()\n        ce = -(targets * logp)                   # per-class CE\n        focal = self.alpha * (1 - p) ** self.gamma * ce\n        loss = focal.sum(dim=1)                  # sum over classes\n        return loss.mean() if self.reduction == \"mean\" else loss.sum()\n\ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n    rot_cleaned = rot_data.copy()\n    for i in range(len(rot_data)):\n        row = rot_data[i]\n        missing_count = np.isnan(row).sum()\n        if missing_count == 0:\n            norm = np.linalg.norm(row)\n            rot_cleaned[i] = row / norm if norm > 1e-8 else [1.0, 0.0, 0.0, 0.0]\n        elif missing_count == 1:\n            missing_idx = np.where(np.isnan(row))[0][0]\n            valid_values = row[~np.isnan(row)]\n            sum_squares = np.sum(valid_values**2)\n            if sum_squares <= 1.0:\n                missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n                if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]) and rot_cleaned[i-1, missing_idx] < 0:\n                    missing_value = -missing_value\n                rot_cleaned[i, missing_idx] = missing_value\n            else:\n                rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n        else:\n            rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n    return rot_cleaned\n\ndef calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n    quat_values = rot_data[:, [1, 2, 3, 0]] if rot_data.shape[1] == 4 else rot_data\n    num_samples = quat_values.shape[0]\n    angular_vel = np.zeros((num_samples, 3))\n    for i in range(num_samples - 1):\n        q_t, q_t_plus_dt = quat_values[i], quat_values[i+1]\n        if np.any(np.isnan(q_t)) or np.any(np.isnan(q_t_plus_dt)):\n            continue\n        try:\n            rot_t, rot_t_plus_dt = R.from_quat(q_t), R.from_quat(q_t_plus_dt)\n            delta_rot = rot_t.inv() * rot_t_plus_dt\n            angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n        except ValueError:\n            pass\n    return angular_vel\n\ndef calculate_angular_distance(rot_data):\n    quat_values = rot_data[:, [1, 2, 3, 0]] if rot_data.shape[1] == 4 else rot_data\n    num_samples = quat_values.shape[0]\n    angular_dist = np.zeros(num_samples)\n    for i in range(num_samples - 1):\n        q1, q2 = quat_values[i], quat_values[i+1]\n        if np.any(np.isnan(q1)) or np.any(np.isnan(q2)):\n            continue\n        try:\n            r1, r2 = R.from_quat(q1), R.from_quat(q2)\n            angle = np.linalg.norm((r1.inv() * r2).as_rotvec())\n            angular_dist[i] = angle\n        except ValueError:\n            pass\n    return angular_dist\n\ndef remove_gravity_from_acc(acc_data, rot_data):\n    acc_values = acc_data.values if isinstance(acc_data, pd.DataFrame) else acc_data\n    quat_values = rot_data.values if isinstance(rot_data, pd.DataFrame) else rot_data\n    quat_scipy = quat_values[:, [1, 2, 3, 0]]\n    num_samples = acc_values.shape[0]\n    linear_accel = np.zeros_like(acc_values)\n    gravity_world = np.array([0, 0, 9.81])\n    for i in range(num_samples):\n        if np.any(np.isnan(quat_scipy[i])):\n            linear_accel[i, :] = acc_values[i, :]\n            continue\n        try:\n            rotation = R.from_quat(quat_scipy[i])\n            gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n            linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n        except ValueError:\n            linear_accel[i, :] = acc_values[i, :]\n    return linear_accel\n\ndef add_features(df: pd.DataFrame) -> pd.DataFrame:\n    df[\"acc_mag\"] = np.linalg.norm(df[Config.ACC_COLS].values, axis=1)\n    df[\"acc_mag_jerk\"] = df.groupby(\"sequence_id\")[\"acc_mag\"].diff().fillna(0)\n    df[\"jerk_x\"], df[\"jerk_y\"], df[\"jerk_z\"] = np.gradient(df[\"acc_x\"]), np.gradient(df[\"acc_y\"]), np.gradient(df[\"acc_z\"])\n    df[\"jerk_magnitude\"] = np.linalg.norm(df[[\"jerk_x\", \"jerk_y\", \"jerk_z\"]].values, axis=1)\n\n    window = 20\n    for _, g in df.groupby(\"sequence_id\"):\n        df.loc[g.index, \"acc_xy_corr\"] = g[\"acc_x\"].rolling(window, min_periods=1).corr(g[\"acc_y\"]).fillna(0)\n        df.loc[g.index, \"acc_xz_corr\"] = g[\"acc_x\"].rolling(window, min_periods=1).corr(g[\"acc_z\"]).fillna(0)\n        df.loc[g.index, \"acc_yz_corr\"] = g[\"acc_y\"].rolling(window, min_periods=1).corr(g[\"acc_z\"]).fillna(0)\n\n    df[\"rot_angle\"] = 2 * np.arccos(df[\"rot_w\"].clip(-1, 1))\n    df[\"rot_angle_vel\"] = df.groupby(\"sequence_id\")[\"rot_angle\"].diff().fillna(0)\n\n    rot_numpy = df[Config.ROT_COLS].to_numpy()\n    angular_vel = calculate_angular_velocity_from_quat(rot_numpy)\n    angular_dist = calculate_angular_distance(rot_numpy)\n    df[[\"angular_vel_x\", \"angular_vel_y\", \"angular_vel_z\"]] = angular_vel\n    df[\"angular_distance\"] = angular_dist\n    df[\"angular_vel_magnitude\"] = np.linalg.norm(angular_vel, axis=1)\n\n    linear_accel = remove_gravity_from_acc(df[Config.ACC_COLS], df[Config.ROT_COLS])\n    df[[\"acc_x2\", \"acc_y2\", \"acc_z2\"]] = linear_accel\n    df[\"acc_mag2\"] = np.linalg.norm(linear_accel, axis=1)\n    df[\"acc_mag_jerk2\"] = df.groupby(\"sequence_id\")[\"acc_mag2\"].diff().fillna(0)\n    df[\"jerk_x2\"], df[\"jerk_y2\"], df[\"jerk_z2\"] = np.gradient(df[\"acc_x2\"]), np.gradient(df[\"acc_y2\"]), np.gradient(df[\"acc_z2\"])\n    df[\"jerk_magnitude2\"] = np.linalg.norm(df[[\"jerk_x2\", \"jerk_y2\", \"jerk_z2\"]].values, axis=1)\n\n    for _, g in df.groupby(\"sequence_id\"):\n        df.loc[g.index, \"acc_xy_corr2\"] = g[\"acc_x2\"].rolling(window, min_periods=1).corr(g[\"acc_y2\"]).fillna(0)\n        df.loc[g.index, \"acc_xz_corr2\"] = g[\"acc_x2\"].rolling(window, min_periods=1).corr(g[\"acc_z2\"]).fillna(0)\n        df.loc[g.index, \"acc_yz_corr2\"] = g[\"acc_y2\"].rolling(window, min_periods=1).corr(g[\"acc_z2\"]).fillna(0)\n\n    df.replace([np.inf, -np.inf], 0, inplace=True)\n    df.fillna(0, inplace=True)\n    return df\n\ndef pad_sequences(sequences: list, maxlen: int, padding: str=\"pre\", truncating: str=\"pre\", dtype: str=\"float32\") -> np.ndarray:\n    n_samples = len(sequences)\n    sample_shape = tuple()\n    for s in sequences:\n        if len(s) > 0:\n            sample_shape = np.asarray(s).shape[1:]\n            break\n    x = np.zeros((n_samples, maxlen) + sample_shape, dtype=dtype)\n    for idx, s in enumerate(sequences):\n        if len(s) == 0:\n            continue\n        s_np = np.asarray(s, dtype=dtype)\n        if truncating == \"pre\":\n            s_np = s_np[-maxlen:]\n        else:\n            s_np = s_np[:maxlen]\n        trunc = s_np.shape[0]\n        if padding == \"pre\":\n            x[idx, -trunc:] = s_np\n        else:\n            x[idx, :trunc] = s_np\n    return x\n\ndef to_categorical(y, num_classes=None):\n    y = np.array(y, dtype=\"int\")\n    if not num_classes:\n        num_classes = np.max(y) + 1\n    return np.eye(num_classes)[y]\n\ndef preprocess_left_handed_pd(df_pd: pd.DataFrame, demo_pd: pd.DataFrame) -> pd.DataFrame:\n    pl_df = pl.DataFrame(df_pd)\n    pl_demo = pl.DataFrame(demo_pd)\n    l_subjs = pl_demo.filter(pl.col(\"handedness\") == 0)[\"subject\"].to_list()\n    r_tr = pl_df.filter(~pl.col(\"subject\").is_in(l_subjs))\n    l_tr = pl_df.filter(pl.col(\"subject\").is_in(l_subjs))\n    if l_tr.shape[0] == 0:\n        return r_tr.to_pandas()\n\n    l_tr = l_tr.with_columns(-pl.col(\"acc_y\"))\n    l_tr = l_tr.with_columns(-pl.col(\"acc_z\"))\n    rot_np = l_tr.select(Config.ROT_COLS).to_numpy()\n    rot_scipy = rot_np[:, [1, 2, 3, 0]]\n    r = R.from_quat(rot_scipy)\n    tmp = r.as_euler(\"xyz\")\n    tmp[:, 1] = -tmp[:, 1]\n    tmp[:, 2] = -tmp[:, 2]\n    r = R.from_euler(\"xyz\", tmp)\n    tmp = r.as_quat()\n    l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n    pl_df2 = pl.concat([r_tr, l_tr]).sort(by=\"row_id\")\n    return pl_df2.to_pandas()\n\n# =============================================================================\n# 3) AUGMENTATIONS\n# =============================================================================\nclass SignalTransform:\n    def __init__(self, p=0.5): self.p = p\n    def __call__(self, y: np.ndarray):\n        if np.random.rand() < self.p: return self.apply(y)\n        return y\n    def apply(self, y: np.ndarray): raise NotImplementedError\n\nclass TimeStretch(SignalTransform):\n    def __init__(self, max_rate=1.5, min_rate=0.5, p=0.5):\n        super().__init__(p); self.max_rate, self.min_rate = max_rate, min_rate\n    def apply(self, x: np.ndarray):\n        rate = np.random.uniform(self.min_rate, self.max_rate)\n        L = x.shape[0]; L_new = int(L / rate)\n        orig_idx = np.linspace(0, L - 1, L)\n        new_idx = np.linspace(0, L - 1, L_new)\n        stretched = np.stack([np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])], axis=1)\n        if L_new < L:\n            padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n            padded[-L_new:, :] = stretched\n            return padded\n        return stretched[-L:, :]\n\nclass TimeShift(SignalTransform):\n    def __init__(self, p=0.5, max_shift_pct=0.25):\n        super().__init__(p); self.max_shift_pct = max_shift_pct\n    def apply(self, x: np.ndarray):\n        L = x.shape[0]\n        shift = np.random.randint(-int(L * self.max_shift_pct), int(L * self.max_shift_pct) + 1)\n        augmented = np.roll(x, shift, axis=0)\n        if shift > 0: augmented[:shift, :] = 0\n        elif shift < 0: augmented[shift:, :] = 0\n        return augmented\n\nclass SignalPermutation(SignalTransform):\n    def __init__(self, p=0.5):\n        super().__init__(p)\n    def apply(self, x: np.ndarray):\n        augmented = x.copy()\n        # acc: cột 0..2\n        perm_acc = np.random.permutation(3)\n        augmented[:, 0:3] = augmented[:, perm_acc]\n        # gyro: ví dụ ở cột 15..17\n        if x.shape[1] > 17:\n            perm_gyro = np.random.permutation(3)\n            original = augmented[:, 15:18].copy()\n            augmented[:, 15:18] = original[:, perm_gyro]\n        return augmented\n\nclass Compose:\n    def __init__(self, transforms: list): self.transforms = transforms\n    def __call__(self, y: np.ndarray):\n        for trns in self.transforms: y = trns(y)\n        return y\n\n\n# --- Pyramid Temporal Feature (PTF) ---\nclass PyramidTemporalFeature(nn.Module):\n    \"\"\"\n    Module trích xuất đặc trưng theo nhiều tần suất thời gian khác nhau.\n    Tương tự FPN nhưng dùng nhiều Conv1D kernel size khác nhau để nắm bắt pattern ngắn & dài.\n    \"\"\"\n    def __init__(self, in_channels, out_channels, kernel_sizes=[3, 5, 7]):\n        super().__init__()\n        self.branches = nn.ModuleList([\n            nn.Sequential(\n                nn.Conv1d(in_channels, out_channels, kernel_size=k, padding=k//2),\n                nn.BatchNorm1d(out_channels),\n                nn.ReLU(inplace=True)\n            )\n            for k in kernel_sizes\n        ])\n        # Gộp lại các scale\n        self.fuse = nn.Conv1d(out_channels * len(kernel_sizes), out_channels, kernel_size=1)\n\n    def forward(self, x):\n        # x: [B, C, T]\n        features = [branch(x) for branch in self.branches]  # nhiều scale thời gian\n        concat = torch.cat(features, dim=1)  # ghép theo kênh\n        out = self.fuse(concat)              # hợp nhất lại\n        return out                           # [B, out_channels, T]\n# =============================================================================\n# 4) DATASET\n# =============================================================================\nclass MixupDataset(Dataset):\n    def __init__(self, X, y, transforms, alpha=0.2, mode=\"train\"):\n        self.X, self.y = X, y\n        self.transforms = transforms\n        self.alpha, self.mode = alpha, mode\n    def __len__(self): return len(self.X)\n    def __getitem__(self, idx):\n        X, y = self.X[idx].copy(), self.y[idx].copy()\n        if self.mode == \"train\":\n            if self.alpha > 0:\n                lam = np.random.beta(self.alpha, self.alpha)\n                idx2 = np.random.randint(len(self.X))\n                X = lam * X + (1 - lam) * self.X[idx2]\n                y = lam * y + (1 - lam) * self.y[idx2]\n            X = self.transforms(X)\n        return torch.FloatTensor(X), torch.FloatTensor(y)\n\n# =============================================================================\n# 5) MODEL\n# =============================================================================\nclass SEBlock(nn.Module):\n    def __init__(self, channel, reduction=8):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True),\n            nn.Linear(channel // reduction, channel), nn.Sigmoid())\n    def forward(self, x):\n        b, c, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1)\n        return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=\"same\", bias=False)\n        self.bn1 = nn.BatchNorm1d(out_channels)\n        self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=\"same\", bias=False)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n        self.se = SEBlock(out_channels)\n        self.relu = nn.ReLU()\n        self.dropout = nn.Dropout(drop)\n        self.pool = nn.MaxPool1d(2)\n        self.shortcut = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, 1, bias=False),\n            nn.BatchNorm1d(out_channels)\n        ) if in_channels != out_channels else nn.Identity()\n    def forward(self, x):\n        identity = self.shortcut(x)\n        out = self.relu(self.bn1(self.conv1(x)))\n        out = self.se(self.bn2(self.conv2(out)))\n        out = self.dropout(self.pool(self.relu(out + identity)))\n        return out\n\nclass AttentionLayer(nn.Module):\n    def __init__(self, hidden_dim): super().__init__(); self.attn = nn.Sequential(nn.Linear(hidden_dim, 1), nn.Tanh())\n    def forward(self, x):\n        weights = F.softmax(self.attn(x).squeeze(-1), dim=1).unsqueeze(-1)\n        return torch.sum(x * weights, dim=1)\n\nclass CrossAttention(nn.Module):\n    def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n        super().__init__()\n        self.feature_dim = feature_dim\n        self.num_heads = num_heads\n        self.head_dim = feature_dim // num_heads\n        assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n        self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n        self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n        self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n        self.out_linear = nn.Linear(feature_dim, feature_dim)\n        self.dropout = nn.Dropout(dropout)\n        self.layer_norm = nn.LayerNorm(feature_dim)\n    def forward(self, query_branch, key_value_branches):\n        B, T, C = query_branch.shape\n        Q = self.q_linear(query_branch)\n        all_kv = torch.cat(key_value_branches, dim=1)\n        K = self.k_linear(all_kv)\n        V = self.v_linear(all_kv)\n        Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2)\n        K = K.view(B, -1, self.num_heads, self.head_dim).transpose(1, 2)\n        V = V.view(B, -1, self.num_heads, self.head_dim).transpose(1, 2)\n        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim)\n        attn_weights = F.softmax(scores, dim=-1)\n        attn_weights = self.dropout(attn_weights)\n        attn_output = torch.matmul(attn_weights, V)\n        attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C)\n        output = self.out_linear(attn_output)\n        output = self.layer_norm(query_branch + output)\n        return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n    def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n        super().__init__()\n        self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n        self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n        self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n    def forward(self, imu1, imu2, imu3):\n        e1 = self.cross_attn1(imu1, [imu2, imu3])\n        e2 = self.cross_attn2(imu2, [imu1, imu3])\n        e3 = self.cross_attn3(imu3, [imu1, imu2])\n        return e1, e2, e3\n\nclass IMUCrossAttentionModel_PTF(nn.Module):\n    def __init__(self, imu_dim, n_classes):\n        super().__init__()\n        self.imu_dim = imu_dim\n\n        # --- 3 nhánh CNN backbone cho dữ liệu IMU ---\n        self.imu_branch1 = nn.Sequential(\n            ResidualSEBlock(12, 128, 3),\n            ResidualSEBlock(128, 256, 5)\n        )\n        self.imu_branch2 = nn.Sequential(\n            ResidualSEBlock(11, 128, 3),\n            ResidualSEBlock(128, 256, 5)\n        )\n        self.imu_branch3 = nn.Sequential(\n            ResidualSEBlock(12, 128, 3),\n            ResidualSEBlock(128, 256, 5)\n        )\n\n        # --- Pyramid Temporal Feature cho mỗi nhánh ---\n        self.ptf1 = PyramidTemporalFeature(256, 128)\n        self.ptf2 = PyramidTemporalFeature(256, 128)\n        self.ptf3 = PyramidTemporalFeature(256, 128)\n\n        # --- Cross attention fusion ---\n        self.cross_attention_fusion = IMUCrossAttentionFusion(feature_dim=128)\n\n        # --- BiLSTM + attention + FC ---\n        self.bilstm = nn.LSTM(128 * 3, 512, bidirectional=True, batch_first=True)\n        self.attention = AttentionLayer(1024)\n        self.fc = nn.Linear(1024, n_classes)\n    \n    def forward(self, x):\n        imu = x[:, :, :self.imu_dim]\n\n        # tách dữ liệu IMU cho 3 nhánh (tuỳ theo cấu trúc cảm biến)\n        imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2))     # [B, C, T]\n        imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2))\n        imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2))\n        \n        # --- Trích đặc trưng đa tần số thời gian ---\n        imu1_ptf = self.ptf1(imu1).transpose(1, 2)\n        imu2_ptf = self.ptf2(imu2).transpose(1, 2)\n        imu3_ptf = self.ptf3(imu3).transpose(1, 2)\n\n        # --- Cross attention fusion giữa 3 nhánh ---\n        imu1, imu2, imu3 = self.cross_attention_fusion(imu1_ptf, imu2_ptf, imu3_ptf)\n    \n        # --- Ghép & đưa qua BiLSTM ---\n        merged = torch.cat((imu1, imu2, imu3), dim=2)\n        lstm_out, _ = self.bilstm(merged)\n\n        # --- Attention + phân lớp ---\n        attended = self.attention(lstm_out)\n        return self.fc(attended)\n\nclass ModelEMA(nn.Module):\n    def __init__(self, model, decay=0.99, device=None):\n        super().__init__()\n        self.module = deepcopy(model)\n        self.module.eval()\n        self.decay = decay\n        self.device = device\n        if self.device:\n            self.module.to(device=device)\n    def _update(self, model, update_fn):\n        with torch.no_grad():\n            for ema_v, model_v in zip(self.module.state_dict().values(), model.state_dict().values()):\n                if self.device:\n                    model_v = model_v.to(device=self.device)\n                ema_v.copy_(update_fn(ema_v, model_v))\n    def update(self, model): self._update(model, update_fn=lambda e, m: self.decay * e + (1. - self.decay) * m)\n    def set(self, model): self._update(model, update_fn=lambda e, m: m)\n\n# =============================================================================\n# 6) TRAIN ONE FOLD\n# =============================================================================\ndef train_single_fold(df_train, df_val, imu_cols, le, fold_num):\n    print(f\"\\n========== Begin Fold {fold_num} | Train IMU Cross-Attention ==========\")\n\n    # pad_len theo percentile của fold\n    seq_lengths = [len(group) for _, group in df_train.groupby(\"sequence_id\")]\n    pad_len = int(np.percentile(seq_lengths, Config.PAD_PERCENTILE))\n\n    # build sequences\n    X_tr_list = [seq[imu_cols].to_numpy() for _, seq in df_train.groupby(\"sequence_id\")]\n    y_tr_int  = [seq[\"gesture_int\"].iloc[0] for _, seq in df_train.groupby(\"sequence_id\")]\n    X_tr = pad_sequences(X_tr_list, maxlen=pad_len)\n    y_tr = to_categorical(y_tr_int, num_classes=len(le.classes_))\n\n    X_va_list = [seq[imu_cols].to_numpy() for _, seq in df_val.groupby(\"sequence_id\")]\n    y_va_int  = [seq[\"gesture_int\"].iloc[0] for _, seq in df_val.groupby(\"sequence_id\")]\n    X_va = pad_sequences(X_va_list, maxlen=pad_len)\n    y_va = to_categorical(y_va_int, num_classes=len(le.classes_))\n\n    model = IMUCrossAttentionModel_PTF(len(imu_cols), len(le.classes_)).to(device)\n    print(f\"Model has {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M parameters.\")\n\n    ema_model = ModelEMA(model, decay=Config.EMA_DECAY, device=device)\n    optimizer = torch.optim.Adam(model.parameters(), lr=Config.LR_INIT, weight_decay=Config.WD)\n    criterion = FocalLoss()\n    scheduler = CosineAnnealingWarmRestarts(optimizer, T_0=Config.T_0)\n\n    transforms = Compose([TimeShift(p=0.35, max_shift_pct=0.25),\n                          TimeStretch(p=0.35),\n                          SignalPermutation(p=0.5)])\n\n    train_dataset = MixupDataset(X_tr, y_tr, alpha=Config.MIXUP_ALPHA, mode=\"train\", transforms=transforms)\n    train_loader = DataLoader(train_dataset, batch_size=Config.BATCH_SIZE, shuffle=True, drop_last=True)\n    val_dataset = MixupDataset(X_va, y_va, alpha=0, mode=\"valid\", transforms=Compose([]))\n    val_loader = DataLoader(val_dataset, batch_size=Config.BATCH_SIZE)\n\n    best_val_loss = float('inf')\n    patience_counter = 0\n    model_path = Config.EXPORT_DIR / f\"model_best_fold_{fold_num}.pt\"\n\n    for epoch in range(Config.EPOCHS):\n        model.train()\n        train_loss = 0.0\n        pbar = tqdm(train_loader, desc=f\"Fold {fold_num} | Epoch {epoch+1}/{Config.EPOCHS}\")\n        for batch_X, batch_y in pbar:\n            batch_X, batch_y = batch_X.to(device), batch_y.to(device)\n            optimizer.zero_grad()\n            outputs = model(batch_X)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()\n            ema_model.update(model)\n            pbar.set_postfix(loss=float(loss.item()))\n        train_loss /= len(train_loader)\n        scheduler.step()\n\n        # validation loss\n        val_loss = 0.0\n        ema_model.module.eval()\n        with torch.no_grad():\n            for batch_X, batch_y in val_loader:\n                out = ema_model.module(batch_X.to(device))\n                val_loss += criterion(out, batch_y.to(device)).item()\n        val_loss /= len(val_loader)\n\n        print(f\"Fold {fold_num} | Epoch [{epoch+1}/{Config.EPOCHS}] Train {train_loss:.4f} | Val {val_loss:.4f}\")\n\n        if val_loss < best_val_loss:\n            best_val_loss = val_loss\n            patience_counter = 0\n            torch.save(ema_model.module.state_dict(), model_path)\n            print(f\"-> Improved Val Loss {best_val_loss:.4f}. Saved to {model_path}\")\n        else:\n            patience_counter += 1\n            if patience_counter >= Config.PATIENCE:\n                print(f\"Early stopping after {epoch+1} epochs.\")\n                break\n\n    ema_model.module.eval()\n    va_probs = []\n    with torch.no_grad():\n        for batch_X, _ in DataLoader(val_dataset, batch_size=Config.BATCH_SIZE):\n            logits = ema_model.module(batch_X.to(device))\n            probs = torch.softmax(logits, dim=1).cpu().numpy()\n            va_probs.append(probs)\n    va_probs = np.vstack(va_probs)                      # (N_val, C)\n    va_preds = va_probs.argmax(1)\n    oof_f1 = f1_score(y_va_int, va_preds, average=\"macro\")\n\n    with open(Config.EXPORT_DIR / f\"oof_score_fold_{fold_num}.json\", \"w\") as f:\n        json.dump({\"fold\": fold_num, \"macro_f1\": float(oof_f1), \"val_loss\": float(best_val_loss)}, f)\n\n    print(f\"Fold {fold_num} done. Best Val Loss: {best_val_loss:.4f} | OOF macro-F1: {oof_f1:.4f}\")\n    return pad_len\n\n# =============================================================================\n# 7) MAIN WORKFLOW \n# =============================================================================\ndef main():\n    print(\"Load data...\")\n    df = pd.read_csv(Config.RAW_DIR / \"train.csv\")\n    dem = pd.read_csv(Config.RAW_DIR / \"train_demographics.csv\")\n\n    if Config.IDS_DEL_PATH.exists():\n        ids2del = pd.read_csv(Config.IDS_DEL_PATH)\n        df = df[~df[\"sequence_id\"].isin(ids2del[\"sequence_id\"])]\n        print(\"Đã loại bỏ các sequence nhiễu từ ids2del.\")\n \n    print(\"✅ Removed sequences successfully.\")\n\n    # labels\n    le = LabelEncoder()\n    df[\"gesture_int\"] = le.fit_transform(df[\"gesture\"])\n    print(\"Quaternions & NaNs...\")\n    df[Config.ROT_COLS] = handle_quaternion_missing_values(df[Config.ROT_COLS].to_numpy())\n    df = pd.concat([seq.ffill().bfill().fillna(0) for _, seq in df.groupby(\"sequence_id\")], axis=0)\n\n    df = preprocess_left_handed_pd(df, dem)\n\n    print(\"Adding features...\")\n    df = add_features(df)\n\n    meta_cols = {\"gesture\", \"gesture_int\", \"sequence_type\", \"behavior\", \"orientation\",\n                 \"row_id\", \"subject\", \"phase\", \"sequence_id\", \"sequence_counter\",\n                 \"handedness\", \"height\", \"weight\", \"age\"}\n    imu_cols = [c for c in df.columns if c not in meta_cols and not (c.startswith(\"thm_\") or c.startswith(\"tof_\"))]\n    print(f\"Sử dụng {len(imu_cols)} IMU features.\")\n\n    sgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n    \n    for fold, (tr_idx, va_idx) in enumerate(sgkf.split(df, y=df[\"gesture_int\"], groups=df[\"subject\"]),start=1):\n        print(f\"\\n--- Processing Fold {fold} ---\")\n        df_train = df.iloc[tr_idx].copy()\n        df_val   = df.iloc[va_idx].copy()\n\n        pad_len = train_single_fold(df_train, df_val, imu_cols, le, fold_num=fold)\n        prep = {\n            \"transformer\": None,\n            \"categories\": list(le.classes_),\n            \"features\": imu_cols,\n            \"max_length\": pad_len,\n            \"fold\": fold\n        }\n        joblib.dump(prep, Config.EXPORT_DIR / f\"prep_fold_{fold}.joblib\")\n        print(f\"Saved prep_fold_{fold}.joblib\")\n\n    print(\"\\n Hoàn tất 5-fold CV.\")\n   \nif __name__ == \"__main__\":\n    main()\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-19T04:01:14.687529Z","iopub.execute_input":"2025-10-19T04:01:14.688130Z","iopub.status.idle":"2025-10-19T04:03:52.508255Z","shell.execute_reply.started":"2025-10-19T04:01:14.688109Z","shell.execute_reply":"2025-10-19T04:03:52.507578Z"}},"outputs":[],"execution_count":null}]}