{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":12944254,"sourceType":"datasetVersion","datasetId":8190684}],"dockerImageVersionId":31090,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom scipy.optimize import minimize_scalar\nimport joblib\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold, GroupKFold\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.metrics import f1_score, confusion_matrix\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, TensorDataset\nimport optuna\nfrom optuna.samplers import TPESampler\nimport json\nimport random\n\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True,max_split_size_mb:64\"\ntorch.backends.cudnn.benchmark = False\ntorch.backends.cudnn.deterministic = True\nacc_cols = ['acc_x', 'acc_y', 'acc_z']\nlin_acc_cols = ['lin_acc_x', 'lin_acc_y', 'lin_acc_z']\nrot_cols = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\nang_vel_cols = ['ang_vel_x', 'ang_vel_y', 'ang_vel_z']\nnorm_cols = ['acc_norm', 'lin_acc_norm', 'ang_vel_norm']\ndelta_cols = ['acc_delta', 'ang_vel_delta', 'lin_acc_delta', 'rot_delta']\nadditional_cols = ['angular_distance', 'acc_mag', 'acc_mag_jerk', 'rot_angle', 'rot_angle_vel']\nthm_cols = [f'thm_{i}' for i in range(1, 6)]\ntof_pixel_cols = [f'tof_{i}_v{j}' for i in range(1, 6) for j in range(64)]\nmodes = [4, 8, 16]\ntof_region_cols = [\n    f\"tof{mode}_{i}_region_{r}_{stat}\"\n    for mode in modes\n    for i in range(1, 6)\n    for r in range(mode)\n    for stat in [\"mean\", \"std\", \"min\", \"max\"]\n]\n\ntof_cols = tof_pixel_cols + tof_region_cols\nimu_cols =  acc_cols + lin_acc_cols + rot_cols + ang_vel_cols + norm_cols + delta_cols + additional_cols\ninput_features = imu_cols + thm_cols + tof_cols","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-02T19:24:58.651170Z","iopub.execute_input":"2025-09-02T19:24:58.651354Z","iopub.status.idle":"2025-09-02T19:25:03.585502Z","shell.execute_reply.started":"2025-09-02T19:24:58.651337Z","shell.execute_reply":"2025-09-02T19:25:03.584940Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_gravity_from_acc(acc_values, rot_values):\n    num_samples = acc_values.shape[0]\n    linear_accel = np.zeros_like(acc_values)\n    gravity_world = np.array([0, 0, 9.81])\n\n    for i in range(num_samples):\n        if np.all(np.isnan(rot_values[i])) or np.all(np.isclose(rot_values[i], 0)):\n            linear_accel[i, :] = acc_values[i, :]\n            continue\n        rotation = R.from_quat(rot_values[i])\n        gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n        linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n    return linear_accel\n\ndef calculate_angular_velocity_from_quat(rot_values, time_delta=1/200): # Assuming 200Hz sampling rate\n    num_samples = rot_values.shape[0]\n    angular_vel = np.zeros((num_samples, 3))\n\n    for i in range(num_samples - 1):\n        q_t = rot_values[i]\n        q_t_plus_dt = rot_values[i+1]\n\n        if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n           np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n            continue\n        rot_t = R.from_quat(q_t)\n        rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n\n        delta_rot = rot_t.inv() * rot_t_plus_dt\n\n        angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n\n    return angular_vel\n\ndef calculate_angular_distance(rot_values):\n    num_samples = rot_values.shape[0]\n    angular_dist = np.zeros(num_samples)\n\n    for i in range(num_samples - 1):\n        q1 = rot_values[i]\n        q2 = rot_values[i+1]\n\n        if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n           np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n            angular_dist[i] = 0\n            continue\n        r1 = R.from_quat(q1)\n        r2 = R.from_quat(q2)\n        relative_rotation = r1.inv() * r2\n        angle = np.linalg.norm(relative_rotation.as_rotvec())\n        angular_dist[i] = angle\n    return angular_dist\n\n\ndef compute_multiscale_tof_stats(df, modes):\n    new_cols = {}\n    for i in range(1, 6):  # tof_1 to tof_5\n        base_cols = [f\"tof_{i}_v{j}\" for j in range(64)]\n        tof_data = df[base_cols].replace(-1, np.nan).copy()\n\n        for mode in modes:\n            region_size = 64 // mode\n            for r in range(mode):\n                region_cols = tof_data.iloc[:, r*region_size:(r+1)*region_size]\n                new_cols[f\"tof{mode}_{i}_region_{r}_mean\"] = region_cols.mean(axis=1)\n                new_cols[f\"tof{mode}_{i}_region_{r}_std\"]  = region_cols.std(axis=1)\n                new_cols[f\"tof{mode}_{i}_region_{r}_min\"]  = region_cols.min(axis=1)\n                new_cols[f\"tof{mode}_{i}_region_{r}_max\"]  = region_cols.max(axis=1)\n\n    return pd.concat([df, pd.DataFrame(new_cols, index=df.index)], axis=1)\n\ndef add_new_features(df):\n    df[lin_acc_cols] = remove_gravity_from_acc(df[acc_cols].values, df[rot_cols].values)\n    df[ang_vel_cols] = calculate_angular_velocity_from_quat(df[rot_cols].values)\n    all_ang_dists = []\n    for _, g in df.groupby('sequence_id', sort=False):\n        ang_dist = calculate_angular_distance(g[rot_cols].values)\n        all_ang_dists.extend(ang_dist)\n    df['angular_distance'] = all_ang_dists\n    df[\"acc_norm\"] = np.linalg.norm(df[acc_cols].values, axis=1)\n    df['lin_acc_norm'] = np.linalg.norm(df[lin_acc_cols], axis=1)\n    df['ang_vel_norm'] = np.linalg.norm(df[ang_vel_cols], axis=1)\n    df[\"acc_delta\"]     = df[\"acc_norm\"].diff().fillna(0)\n    df[\"ang_vel_delta\"] = df[\"ang_vel_norm\"].diff().fillna(0)\n    df[\"lin_acc_delta\"] = df[\"lin_acc_norm\"].diff().fillna(0)\n    df[\"rot_delta\"]     = df[rot_cols].diff().pow(2).sum(axis=1).pow(0.5).fillna(0)\n    df['acc_mag'] = np.linalg.norm(df[acc_cols].values, axis=1)\n    df['rot_angle'] = 2 * np.arccos(np.clip(df['rot_w'], -1, 1))\n    df['acc_mag_jerk'] = df.groupby('sequence_id')['acc_mag'].diff().fillna(0)\n    df['rot_angle_vel'] = df.groupby('sequence_id')['rot_angle'].diff().fillna(0)\n    df = compute_multiscale_tof_stats(df, modes)\n    return df\n\ndef build_tensor(df, feature_cols, sequence_col=\"sequence_id\", cap_len=None, mode=\"train\", rng=None):\n    if rng is None:\n        rng = np.random.default_rng(42)\n    seq_tensors, seq_ids = [], []\n    for sid, g in df.groupby(sequence_col, sort=False):  # keep original order\n        X = g[feature_cols].to_numpy(dtype=np.float32)\n        if cap_len is not None and len(X) > cap_len:\n            if mode == \"train\":\n                start = rng.integers(0, len(X) - cap_len + 1)\n                X = X[start:start+cap_len]\n            else:  # eval: deterministic\n                X = X[:cap_len]\n        seq_tensors.append(torch.from_numpy(X))\n        seq_ids.append(sid)\n    padded = torch.nn.utils.rnn.pad_sequence(seq_tensors, batch_first=True)  # [N, T, D]\n    return padded, np.array(seq_ids)\n\n\ndef group_normalize(df, feature_cols, group_col='sequence_id'):\n    return df.groupby(group_col)[feature_cols].transform(\n        lambda x: (x - x.mean()) / (x.std() + 1e-6)\n    )\n\ndef make_train_data():\n    train = pd.read_csv('cmi-detect-behavior-with-sensor-data/train.csv')\n    train = add_new_features(train)\n    train[input_features] = train[input_features].interpolate(method=\"linear\", limit_direction=\"both\", axis=0)\n    train[input_features] = train[input_features].fillna(0.0)\n\n    scaler = StandardScaler()\n    train[input_features] = scaler.fit_transform(train[input_features])\n    os.makedirs(\"cmi-checkpoint\", exist_ok=True)\n\n    joblib.dump(scaler, \"cmi-checkpoint/scaler.pkl\")\n    train_group = train.copy()\n    train_group[input_features] = group_normalize(train_group, input_features)\n    train_global_tensor, train_seq_order  = build_tensor(train, input_features, cap_len=120, mode='train')\n    train_group_tensor,  train_seq_order2 = build_tensor(train_group, input_features, cap_len=120, mode='train')\n    X_train = torch.cat([train_global_tensor, train_group_tensor], dim=2)  # [N, T, 2*D]\n\n    seq_label_series = (\n    train.groupby('sequence_id', sort=False)['gesture']\n        .apply(lambda s: s.iloc[0])\n    )\n    seq_labels_ordered = seq_label_series.reindex(train_seq_order).values\n    target_labels = sorted(train.loc[train['sequence_type'] == 'Target', 'gesture'].unique())\n    seq_labels_ordered = np.where(\n        np.isin(seq_labels_ordered, target_labels),\n        seq_labels_ordered,\n        -1\n    )\n\n    le = LabelEncoder()\n    le.fit(target_labels)\n    joblib.dump(le, \"cmi-checkpoint/label_encoder.pkl\")\n    mask = seq_labels_ordered != -1\n    y_train = np.full(len(seq_labels_ordered), -1)  # Initialize all as -1\n    y_train[mask] = le.transform(seq_labels_ordered[mask])\n    y_train = torch.tensor(y_train).long()\n    groups = train_seq_order\n    \n    return X_train, y_train, groups, le\n\nclass ResidualSEBlock(nn.Module):\n    def __init__(self, Cin, Cout, kernel_size, dropout, activation, se_reduction, drop_prob):\n        super().__init__()\n        self.conv1 = nn.Conv1d(Cin, Cout, kernel_size, padding=kernel_size // 2)\n        self.norm1 = nn.LayerNorm(Cout)\n        self.conv2 = nn.Conv1d(Cout, Cout, kernel_size, padding=kernel_size // 2)\n        self.norm2 = nn.LayerNorm(Cout)\n        self.activation = activation\n        self.skip = nn.Conv1d(Cin, Cout, 1) if Cin != Cout else nn.Identity()\n        self.se_fc = nn.Sequential(\n            nn.Linear(Cout, Cout // se_reduction, bias=False),\n            activation,\n            nn.Linear(Cout // se_reduction, Cout, bias=False),\n            nn.Sigmoid()\n        )\n        self.se_pool = nn.AdaptiveAvgPool1d(1)\n\n        self.drop_prob = drop_prob\n\n    def stochastic_depth(self, x):\n        if not self.training or self.drop_prob == 0.0:\n            return x\n        keep_prob = 1.0 - self.drop_prob\n        shape = [x.shape[0]] + [1] * (x.ndim - 1)\n        mask = torch.rand(shape, dtype=x.dtype, device=x.device) < keep_prob\n        return x * mask / keep_prob\n\n    def se_block(self, x):\n        b, c, _ = x.size()\n        y = self.se_pool(x).view(b, c)\n        y = self.se_fc(y).view(b, c, 1)\n        return x * y\n\n    def forward(self, x):  # [B, C, T]\n        residual = self.skip(x)\n        out = self.conv1(x)\n        out = self.norm1(out.transpose(1, 2)).transpose(1, 2)  # LayerNorm over channels\n        out = self.activation(out)\n\n        out = self.conv2(out)\n        out = self.norm2(out.transpose(1, 2)).transpose(1, 2)\n\n        out = self.se_block(out)\n\n        out = self.stochastic_depth(out)\n\n        return self.activation(out + residual)\n\n\nclass NoiseDenseBranch(nn.Module):\n    def __init__(self, input_dim, cfg):\n        super().__init__()\n        hidden_dim = cfg.get(\"hidden_dim\", 8)\n        dropout = cfg.get(\"dropout\", 0.1)\n        self.noise_std = cfg.get(\"noise_std\", 0.09)\n\n        self.block = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim),\n            nn.ELU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, hidden_dim)\n        )\n\n    def forward(self, x):  # x: [B, T, D]\n        if self.training and self.noise_std > 0:\n            noise = torch.randn_like(x) * self.noise_std\n            x = x + noise\n        return self.block(x)\n\n\n\nclass MultiQueryAttention(nn.Module):\n    def __init__(self, embed_dim, num_heads, dropout=0.1):\n        super().__init__()\n        self.num_heads = num_heads\n        self.embed_dim = embed_dim\n        self.dropout = dropout\n\n        self.q_proj = nn.Linear(embed_dim, embed_dim)\n        self.k_proj = nn.Linear(embed_dim, embed_dim // num_heads)  # shared KV\n        self.v_proj = nn.Linear(embed_dim, embed_dim // num_heads)\n        self.out_proj = nn.Linear(embed_dim, embed_dim)\n\n    def forward(self, x):  # x: [B, T, D]\n        B, T, D = x.size()\n        H = self.num_heads\n        head_dim = D // H\n\n        q = self.q_proj(x).view(B, T, H, head_dim).transpose(1, 2)  # [B, H, T, d]\n        k = self.k_proj(x).mean(dim=1, keepdim=True)                # [B, 1, d]\n        v = self.v_proj(x).mean(dim=1, keepdim=True)                # [B, 1, d]\n\n        k = k.unsqueeze(1).expand(-1, H, -1, -1)  # [B, H, 1, d]\n        v = v.unsqueeze(1).expand(-1, H, -1, -1)  # [B, H, 1, d]\n\n        attn_scores = torch.matmul(q, k.transpose(-2, -1)) / (head_dim ** 0.5)  # [B, H, T, 1]\n        attn_weights = torch.softmax(attn_scores, dim=-2)\n        attn_weights = F.dropout(attn_weights, p=self.dropout, training=self.training)\n\n        attn_output = torch.matmul(attn_weights, v)  # [B, H, T, d]\n        attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, D)\n\n        return self.out_proj(attn_output)  # [B, T, D]\n\n\nclass MultiQueryTransformerBlock(nn.Module):\n    def __init__(self, input_dim, cfg, activation):\n        super().__init__()\n        num_heads = cfg.get(\"num_heads\", 4)\n        embed_dim = cfg.get(\"embed_dim\", 512)\n        attn_dropout = cfg.get(\"attn_dropout\", 0.1)\n        trans_dropout = cfg.get(\"trans_dropout\", 0.2)\n\n        self.norm0 = nn.LayerNorm(input_dim)\n        self.input_proj = nn.Linear(input_dim, embed_dim) if input_dim != embed_dim else nn.Identity()\n        self.attn = MultiQueryAttention(embed_dim, num_heads, dropout=attn_dropout)\n        self.norm1 = nn.LayerNorm(embed_dim)\n        self.dropout1 = nn.Dropout(trans_dropout)\n\n        ff_hidden_dim = cfg.get(\"ff_hidden_dim\", input_dim * 2)\n        ff_num_layers = cfg.get(\"ff_num_layers\", 1)\n        ff_dropout = cfg.get(\"ff_dropout\", 0.1)\n\n\n        def make_ff(input_dim, num_layers, hidden_dim, dropout, activation):\n            layers = []\n            in_dim = input_dim\n            for _ in range(num_layers):\n                layers += [\n                    nn.Linear(in_dim, hidden_dim),\n                    activation,\n                    nn.Dropout(dropout)\n                ]\n                in_dim = hidden_dim\n            layers.append(nn.Linear(hidden_dim, input_dim))\n            return nn.Sequential(*layers)\n\n        self.ffn = make_ff(embed_dim, ff_num_layers, ff_hidden_dim, ff_dropout, activation)\n\n        self.norm2 = nn.LayerNorm(embed_dim)\n        self.dropout2 = nn.Dropout(trans_dropout)\n\n        self.reverse_mask_drop_prob = cfg.get(\"reverse_mask_drop_prob\", 0.15)\n\n        self.attn_score = nn.Linear(embed_dim, 1)\n        self.gate = nn.Parameter(torch.zeros(embed_dim))  # initialized equally\n\n    def reverse_mask_attention(self, x, drop_prob=0.0):\n        if not self.training or drop_prob == 0.0:\n            return x\n        B, T, D = x.shape\n        mask = torch.rand(B, T, device=x.device) > drop_prob\n        return x * mask.unsqueeze(-1)\n\n    def forward(self, x):  # x: [B, T, D]\n        x = self.input_proj(self.norm0(x))\n        x = x + self.dropout1(self.attn(self.norm1(x)))\n        x = x + self.dropout2(self.ffn(self.norm2(x)))\n\n        masked_out = self.reverse_mask_attention(x, drop_prob=self.reverse_mask_drop_prob)\n        attn_weights = torch.softmax(self.attn_score(masked_out), dim=1)  # [B, T, 1]\n        weighted = torch.sum(x * attn_weights, dim=1)  # [B, D]\n        residual = self.norm2(x).mean(dim=1)  # [B, D]\n        pooled = torch.sigmoid(self.gate) * weighted + (1 - torch.sigmoid(self.gate)) * residual\n\n        return pooled  # [B, D]\n\n\nclass CNNTwoBranchModel(nn.Module):\n    def __init__(self, imu_dim, tof_thm_dim, imu_cnn_cfg, tof_thm_cnn_cfg, noise_dense_branch_cfg,\n                 lstm_cfg, gru_cfg, trans_cfg, gesture_head_cfg,\n                 binary_head_cfg, activation_cfg, num_classes, training=True):\n        super().__init__()\n        self.imu_dim = imu_dim\n        self.tof_thm_dim = tof_thm_dim\n        self.training = training\n\n        def get_activation(name):\n            if name is None:\n                return nn.SiLU()\n            return {\n                \"silu\": nn.SiLU(),\n                \"gelu\": nn.GELU(),\n                \"relu\": nn.ReLU()\n            }[name]\n\n        def make_cnn(Cin, cfg, activation):\n            hidden_dim = cfg.get(\"hidden_dim\", 512)\n            kernel_size = cfg.get(\"kernel_size\", 5)\n            num_layers = cfg.get(\"num_layers\", 2)\n            se_reduction = cfg.get(\"se_reduction\", 8)\n            drop_prob = cfg.get(\"drop_prob\", 0.0)\n            base_dropout = 0.1\n\n            layers = []\n            for i in range(num_layers):\n                in_dim = Cin if i == 0 else hidden_dim\n                dropout = base_dropout if i == num_layers - 1 else 0.0\n                layers.append(\n                    ResidualSEBlock(\n                        in_dim, hidden_dim, kernel_size,\n                        dropout, activation, se_reduction, drop_prob\n                    )\n                )\n\n            return nn.Sequential(*layers)\n\n        cnn_activation = get_activation(activation_cfg[\"cnn\"])\n        self.imu_cnn = make_cnn(imu_dim, imu_cnn_cfg, cnn_activation)\n        self.tof_cnn = make_cnn(tof_thm_dim, tof_thm_cnn_cfg, cnn_activation)\n\n        concat_cnn_out_dim = imu_cnn_cfg[\"hidden_dim\"] * 2 + tof_thm_cnn_cfg[\"hidden_dim\"] * 2\n\n        self.noise_dense_branch = NoiseDenseBranch(concat_cnn_out_dim, noise_dense_branch_cfg)\n\n        rnn_input_dim = concat_cnn_out_dim + noise_dense_branch_cfg[\"hidden_dim\"]\n\n        self.bilstm = nn.LSTM(\n            input_size=rnn_input_dim,\n            hidden_size=lstm_cfg[\"hidden_dim\"],\n            num_layers=lstm_cfg[\"num_layers\"],\n            dropout=lstm_cfg[\"dropout\"] if lstm_cfg[\"num_layers\"] > 1 else 0.0,\n            bidirectional=True,\n            batch_first=True\n        )\n\n        self.bigru = nn.GRU(\n            input_size=rnn_input_dim,\n            hidden_size=gru_cfg[\"hidden_dim\"],\n            num_layers=gru_cfg[\"num_layers\"],\n            dropout=gru_cfg[\"dropout\"] if gru_cfg[\"num_layers\"] > 1 else 0.0,\n            bidirectional=True,\n            batch_first=True\n        )\n\n        concat_rnn_dense_dim = (\n            lstm_cfg[\"hidden_dim\"] * 2 +\n            gru_cfg[\"hidden_dim\"] * 2\n        )\n        ffn_activation = get_activation(activation_cfg[\"ffn\"])\n        self.trans = MultiQueryTransformerBlock(concat_rnn_dense_dim, trans_cfg, ffn_activation)\n\n        def make_head(input_dim, head_cfg, num_classes, activation):\n            layers = []\n            hidden_dim = head_cfg.get(\"hidden_dim\", 128)\n            num_layers = head_cfg.get(\"num_layers\", 1)\n            dropout = head_cfg.get(\"dropout\", 0.1)\n            for _ in range(num_layers - 1):\n                layers += [\n                    nn.Linear(input_dim, hidden_dim),\n                    activation,\n                    nn.Dropout(dropout)\n                ]\n                input_dim = hidden_dim\n            layers.append(nn.Linear(input_dim, num_classes))\n            return nn.Sequential(*layers)\n\n        head_activation = get_activation(activation_cfg[\"head\"])\n        head_input_dim = trans_cfg['embed_dim']\n        self.gesture_head = make_head(head_input_dim , gesture_head_cfg, num_classes, head_activation)\n        self.binary_head = make_head(head_input_dim , binary_head_cfg, 1, head_activation)\n\n    def forward(self, x):  # x: [B, T, 2*(imu_dim + tof_dim)]\n        D_imu, D_tof = self.imu_dim, self.tof_thm_dim\n        total_dim = 2 * (D_imu + D_tof)\n\n        x_imu_global = x[..., :D_imu]                            # [B, T, D_imu]\n        x_tof_global = x[..., D_imu:D_imu+D_tof]                 # [B, T, D_tof]\n        x_imu_group  = x[..., D_imu+D_tof:D_imu*2+D_tof]         # [B, T, D_imu]\n        x_tof_group  = x[..., D_imu*2+D_tof:]                    # [B, T, D_tof]\n\n        imu_global_out = self.imu_cnn(x_imu_global.permute(0, 2, 1))  # [B, H, T]\n        imu_group_out  = self.imu_cnn(x_imu_group.permute(0, 2, 1))   # same weights\n\n        tof_global_out = self.tof_cnn(x_tof_global.permute(0, 2, 1))  # [B, H, T]\n        tof_group_out  = self.tof_cnn(x_tof_group.permute(0, 2, 1))   # same weights\n\n        x_cat_cnn = torch.cat([imu_global_out, imu_group_out, tof_global_out, tof_group_out], dim=1)             # [B, 4H, T]\n        x_seq_cnn = x_cat_cnn.permute(0, 2, 1)                           # [B, T, 4H]\n\n        x_noise   = self.noise_dense_branch(x_seq_cnn)  # [B, T, 16]\n        x_seq_cnn = torch.cat([x_seq_cnn, x_noise], dim=2)\n        x_lstm, _ = self.bilstm(x_seq_cnn)\n        x_gru, _  = self.bigru(x_seq_cnn)\n        x_cat_rnn = torch.cat([x_lstm, x_gru], dim=2)\n\n        x_trans = self.trans(x_cat_rnn)       # [B, total_dim]\n\n        gesture_logits = self.gesture_head(x_trans)  # [B, num_classes]\n        binary_logits  = self.binary_head(x_trans)   # [B, 1]\n\n        return gesture_logits, binary_logits\n\n\nclass EarlyStopper():\n    def __init__(self, patience=5, min_delta=1e-4, alpha=0.6):\n        self.patience = patience\n        self.min_delta = min_delta\n        self.alpha = alpha\n        self.best = -float(\"inf\")\n        self.wait = 0\n        self.best_state = None\n        self.ema_score = None\n\n    def step(self, score, model):\n        self.ema_score = score if self.ema_score is None else self.alpha * score + (1 - self.alpha) * self.ema_score\n        if self.ema_score > self.best + self.min_delta:\n            self.best = self.ema_score\n            self.wait = 0\n            self.best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}\n            return False\n        self.wait += 1\n        return self.wait >= self.patience\n\n\nclass SpecAugment(nn.Module):\n    def __init__(self, max_time_width=10, max_feature_width=5, prob=0.3):\n        super().__init__()\n        self.max_time_width = max_time_width\n        self.max_feature_width = max_feature_width\n        self.prob = prob\n\n    def forward(self, x):\n        if not self.training or np.random.rand() > self.prob:\n            return x\n\n        B, T, D = x.shape\n        x_aug = x.clone()\n\n        for b in range(B):\n            if self.max_time_width > 0:\n                t0 = np.random.randint(0, T)\n                t_width = np.random.randint(1, self.max_time_width + 1)\n                t1 = min(T, t0 + t_width)\n                x_aug[b, t0:t1, :] = 0.0\n\n            if self.max_feature_width > 0:\n                f0 = np.random.randint(0, D)\n                f_width = np.random.randint(1, self.max_feature_width + 1)\n                f1 = min(D, f0 + f_width)\n                x_aug[b, :, f0:f1] = 0.0\n\n        return x_aug\n\nclass FocalLoss(nn.Module):\n    def __init__(self, gamma=2.0, weight=None, reduction='mean'):\n        super().__init__()\n        self.gamma = gamma\n        self.weight = weight\n        self.reduction = reduction\n\n    def forward(self, input, target):\n        logp = F.log_softmax(input, dim=1)\n        p = logp.exp()\n        logp = logp.gather(1, target.unsqueeze(1)).squeeze(1)\n        p = p.gather(1, target.unsqueeze(1)).squeeze(1)\n\n        loss = -(1 - p) ** self.gamma * logp\n        if self.weight is not None:\n            w = self.weight[target]\n            loss = loss * w\n\n        if self.reduction == 'mean':\n            return loss.mean()\n        elif self.reduction == 'sum':\n            return loss.sum()\n        else:\n            return loss\n\n\ndef create_loader(X, y, batch_size, shuffle=True):\n    ds = TensorDataset(X, y)\n    return DataLoader(ds, batch_size=batch_size, shuffle=shuffle)\n\ndef mixup_data(x, y, alpha=0.4):\n    if alpha > 0.0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1.0\n\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(x.device)\n\n    mixed_x = lam * x + (1 - lam) * x[index]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\n\ndef competition_f1(y_true, y_pred):\n    y_true_bin = np.array([1 if y != -1 else 0 for y in y_true])\n    y_pred_bin = np.array([1 if y != -1 else 0 for y in y_pred])\n    binary_f1 = f1_score(y_true_bin, y_pred_bin)\n\n    all_labels = np.unique(np.concatenate([y_true, y_pred]))\n    macro_f1 = f1_score(y_true, y_pred, labels=all_labels, average=\"macro\")\n\n    return 0.5 * binary_f1 + 0.5 * macro_f1, binary_f1, macro_f1\n\ndef stratified_split_with_nontarget(\n    X, y, groups, n_splits=5, random_state=42\n):\n    mask_target = y != -1\n    mask_nontarget = ~mask_target\n\n    target_indices = np.arange(len(X))[mask_target]\n    nontarget_indices = np.arange(len(X))[mask_nontarget]\n\n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=random_state)\n    gkf  = GroupKFold(n_splits=n_splits)\n    target_folds = list(sgkf.split(target_indices, y[mask_target].numpy(), groups[mask_target]))\n    nontarget_folds = list(gkf.split(nontarget_indices, np.zeros(len(nontarget_indices)), groups[mask_nontarget]))\n\n    for fold in range(n_splits):\n        target_train_idx, target_val_idx = target_folds[fold]\n        nt_train_idx, nt_val_idx = nontarget_folds[fold]\n        target_train_idx = target_indices[target_train_idx]\n        target_val_idx   = target_indices[target_val_idx]\n        nt_train_idx     = nontarget_indices[nt_train_idx]\n        nt_val_idx       = nontarget_indices[nt_val_idx]\n        train_idx = np.concatenate([target_train_idx, nt_train_idx])\n        val_idx   = np.concatenate([target_val_idx, nt_val_idx])\n\n        yield train_idx, val_idx\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam):\n    mask_a = y_a != -1\n    mask_b = y_b != -1\n    mask = mask_a & mask_b\n    if mask.sum() == 0:\n        return torch.tensor(0.0, device=pred.device)\n    return lam * criterion(pred[mask], y_a[mask]) + (1 - lam) * criterion(pred[mask], y_b[mask])\n\ndef dual_margin_loss(logits, targets, pos_margin=0.3, neg_margin=0.1):\n    B, C = logits.shape\n    true_logits = logits.gather(1, targets.unsqueeze(1)).squeeze(1)  # [B]\n\n    one_hot = F.one_hot(targets, num_classes=C).bool()\n    logits_masked = logits.masked_fill(one_hot, float('-inf'))\n\n    max_other_logits = logits_masked.max(dim=1).values\n    inter_margin_loss = F.relu(pos_margin - (true_logits - max_other_logits))\n\n    mean_other_logits = logits_masked.masked_fill(torch.isnan(logits_masked), 0).mean(dim=1)\n    intra_margin_loss = F.relu(neg_margin - (true_logits - mean_other_logits))\n\n    return inter_margin_loss.mean() + 0.05 * intra_margin_loss.mean()\n\ndef optimize_binary_threshold(binary_probs, gesture_probs, labels):\n    thresholds = np.linspace(0.05, 0.95, 50)\n    best_score, best_thresh = -1, 0.5\n    for th in thresholds:\n        preds = np.full(len(labels), -1)\n        mask = binary_probs >= th\n        preds[mask] = np.argmax(gesture_probs[mask], axis=1)\n        score, _, _ = competition_f1(labels, preds)\n        if score > best_score:\n            best_score, best_thresh = score, th\n    return best_thresh","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T19:25:03.586861Z","iopub.execute_input":"2025-09-02T19:25:03.587711Z","iopub.status.idle":"2025-09-02T19:25:03.647873Z","shell.execute_reply.started":"2025-09-02T19:25:03.587690Z","shell.execute_reply":"2025-09-02T19:25:03.647167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_model():\n    X_train, y_train, groups, le = make_train_data()\n    num_classes = len(le.classes_)\n    rng = np.random.default_rng(seed=42)   \n    unique_groups = np.unique(groups)\n    tune_groups = rng.choice(unique_groups, size=int(len(unique_groups)*0.3), replace=False)\n    tune_mask = np.isin(groups, tune_groups)\n    X_tune, y_tune, groups_tune = X_train[tune_mask], y_train[tune_mask], groups[tune_mask]\n    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    sampler = TPESampler(seed=42, multivariate=True, warn_independent_sampling=False)\n    pruner  = optuna.pruners.MedianPruner(n_startup_trials=10, n_warmup_steps=0)\n    study_name = \"2branch_2head_study\"\n    study = optuna.create_study(\n        study_name=study_name,\n        storage=f\"sqlite:///optuna_{study_name}.db\",\n        direction=\"minimize\",\n        load_if_exists=True,\n        sampler=sampler,\n        pruner=pruner\n    )\n    study.optimize(objective, n_trials=100, n_jobs=1)\n    print(\"Best params:\", study.best_params)\n    print(\"Best CV correlation:\", study.best_value)\n    best_trial = study.best_trial\n    best_params = {\n        \"imu_cnn_cfg\": {\n            \"hidden_dim\": best_trial.params[\"imu_cnn_hidden_dim\"],\n            \"kernel_size\": best_trial.params[\"imu_cnn_kernel_size\"],\n            \"num_layers\": best_trial.params[\"imu_cnn_num_layers\"],\n            \"se_reduction\": best_trial.params[\"imu_cnn_se_reduction\"],\n            \"drop_prob\": best_trial.params[\"imu_cnn_drop_prob\"]\n        },\n        \"tof_thm_cnn_cfg\": {\n            \"hidden_dim\": best_trial.params[\"tof_thm_cnn_hidden_dim\"],\n            \"kernel_size\": best_trial.params[\"tof_thm_cnn_kernel_size\"],\n            \"num_layers\": best_trial.params[\"tof_thm_cnn_num_layers\"],\n            \"se_reduction\": best_trial.params[\"tof_thm_se_reduction\"],\n            \"drop_prob\": best_trial.params[\"tof_thm_cnn_drop_prob\"]\n        },\n        \"noise_dense_branch_cfg\": {\n            \"hidden_dim\": best_trial.params[\"noise_dense_hidden_dim\"],\n            \"dropout\": best_trial.params[\"noise_dense_dropout\"],\n            \"noise_std\": best_trial.params[\"noise_dense_noise_std\"]\n        },\n        \"lstm_cfg\": {\n            \"hidden_dim\": best_trial.params[\"lstm_hidden_dim\"],\n            \"num_layers\": best_trial.params[\"lstm_num_layers\"],\n            \"dropout\": best_trial.params[\"lstm_dropout\"]\n        },\n        \"gru_cfg\": {\n            \"hidden_dim\": best_trial.params[\"gru_hidden_dim\"],\n            \"num_layers\": best_trial.params[\"gru_num_layers\"],\n            \"dropout\": best_trial.params[\"gru_dropout\"]\n        },\n        \"trans_cfg\": {\n            \"num_heads\": best_trial.params[\"attn_num_heads\"],\n            \"embed_dim\": best_trial.params[\"attn_embed_dim\"],\n            \"attn_dropout\": best_trial.params[\"attn_dropout\"],\n            \"trans_dropout\": best_trial.params[\"trans_dropout\"],\n            \"ff_hidden_dim\": best_trial.params[\"ff_hidden_dim\"],\n            \"ff_num_layers\": best_trial.params[\"ff_num_layers\"],\n            \"ff_dropout\": best_trial.params[\"ff_dropout\"],\n            \"reverse_mask_drop_prob\": best_trial.params[\"attn_reverse_mask_drop_prob\"]\n        },\n        \"gesture_head_cfg\": {\n            \"hidden_dim\": best_trial.params[\"gesture_hidden_dim\"],\n            \"num_layers\": best_trial.params[\"gesture_num_layers\"],\n            \"dropout\": best_trial.params[\"gesture_dropout\"]\n        },\n        \"binary_head_cfg\": {\n            \"hidden_dim\": best_trial.params[\"binary_hidden_dim\"],\n            \"num_layers\": best_trial.params[\"binary_num_layers\"],\n            \"dropout\": best_trial.params[\"binary_dropout\"]\n        },\n        \"activation_cfg\": {\n            \"cnn\": \"silu\",\n            \"ffn\": \"relu\",\n            \"head\": \"gelu\"\n        },\n        \"specaug_cfg\": {\n            \"max_time_width\": best_trial.params[\"specaug_time_width\"],\n            \"max_feature_width\": best_trial.params[\"specaug_feature_width\"],\n            \"prob\": best_trial.params[\"specaug_prob\"]\n        },\n        \"batch_size\": best_trial.params[\"batch_size\"],\n        \"lr\": best_trial.params[\"lr\"],\n        \"weight_decay\": best_trial.params[\"weight_decay\"],\n        \"focal_gamma\": best_trial.params[\"focal_gamma\"],\n        \"epochs\": best_trial.params[\"epochs\"],\n        \"mixup_alpha\": best_trial.params[\"mixup_alpha\"],\n        \"gesture_logit_margin\": best_trial.params[\"gesture_logit_margin\"],\n        \"gesture_neg_margin\": best_trial.params[\"gesture_neg_margin\"],\n        \"margin_weight\": best_trial.params[\"margin_weight\"],\n        \"penalty_weight\": best_trial.params[\"penalty_weight\"],\n        \"conf_penalty_update_epoch\": best_trial.params[\"conf_penalty_update_epoch\"],\n        \"gesture_ratio\": best_trial.params[\"gesture_ratio\"]\n        }\n    with open(os.path.join(save_dir, \"best_params.json\"), \"w\") as f:\n        json.dump(best_params, f, indent=2)\n    train_and_save_model_per_fold(best_params, X_train, y_train, groups, save_dir=\"/kaggle/working/fold_models\", n_splits=5)\n    train_full_model(best_params, X_train, y_train, save_dir=\"/kaggle/working/full_models\")\n\n    def objective(trial):\n        imu_cnn_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"imu_cnn_hidden_dim\", 256, 1280, step=256),\n            \"kernel_size\": trial.suggest_int(\"imu_cnn_kernel_size\", 3, 9, step=2),\n            \"num_layers\": trial.suggest_int(\"imu_cnn_num_layers\", 2, 4),\n            \"se_reduction\": trial.suggest_int(\"imu_cnn_se_reduction\", 4, 18, step=2),\n            \"drop_prob\": trial.suggest_float(\"imu_cnn_drop_prob\", 0.10, 0.40)\n        }\n        tof_thm_cnn_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"tof_thm_cnn_hidden_dim\", 256, 1024, step=256),\n            \"kernel_size\": trial.suggest_int(\"tof_thm_cnn_kernel_size\", 3, 9, step=2),\n            \"num_layers\": trial.suggest_int(\"tof_thm_cnn_num_layers\", 1, 3),\n            \"se_reduction\": trial.suggest_int(\"tof_thm_se_reduction\", 2, 10, step=2),\n            \"drop_prob\": trial.suggest_float(\"tof_thm_cnn_drop_prob\", 0.10, 0.40)\n        }\n        noise_dense_branch_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"noise_dense_hidden_dim\", 8, 32, step=8),\n            \"dropout\": trial.suggest_float(\"noise_dense_dropout\", 0.05, 0.2),\n            \"noise_std\": trial.suggest_float(\"noise_dense_noise_std\", 0.05, 0.15),\n        }\n        lstm_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"lstm_hidden_dim\", 128, 512, step=128),\n            \"num_layers\": trial.suggest_int(\"lstm_num_layers\", 1, 3),\n            \"dropout\": trial.suggest_float(\"lstm_dropout\", 0.15, 0.45)\n        }\n        gru_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"gru_hidden_dim\", 128, 512, step=128),\n            \"num_layers\": trial.suggest_int(\"gru_num_layers\", 1, 3),\n            \"dropout\": trial.suggest_float(\"gru_dropout\", 0.1, 0.35)\n        }\n\n        trans_cfg = {\n            \"num_heads\": trial.suggest_categorical(\"attn_num_heads\", [2, 4, 8, 16]),\n            \"embed_dim\" : trial.suggest_int(\"attn_embed_dim\", 256, 1024, step=256),\n            \"attn_dropout\": trial.suggest_float(\"attn_dropout\", 0.05, 0.4),\n            \"trans_dropout\": trial.suggest_float(\"trans_dropout\", 0.1, 0.4),\n            \"ff_hidden_dim\": trial.suggest_int(\"ff_hidden_dim\", 256, 1024, step=256),\n            \"ff_num_layers\": trial.suggest_int(\"ff_num_layers\", 3, 8),\n            \"ff_dropout\": trial.suggest_float(\"ff_dropout\", 0.05, 0.4),\n            \"reverse_mask_drop_prob\": trial.suggest_float(\"attn_reverse_mask_drop_prob\", 0.0, 0.3)\n        }\n        gesture_head_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"gesture_hidden_dim\", 256, 1024, step=256),\n            \"num_layers\": trial.suggest_int(\"gesture_num_layers\", 1, 4),\n            \"dropout\": trial.suggest_float(\"gesture_dropout\", 0.15, 0.5)\n        }\n        binary_head_cfg = {\n            \"hidden_dim\": trial.suggest_int(\"binary_hidden_dim\", 128, 256, step=64),\n            \"num_layers\": trial.suggest_int(\"binary_num_layers\", 1, 3),\n            \"dropout\": trial.suggest_float(\"binary_dropout\", 0.05, 0.25)\n        }\n        activation_cfg = {\n            \"cnn\": trial.suggest_categorical(\"cnn_activation\", [\"relu\", \"silu\"]),\n            \"ffn\": trial.suggest_categorical(\"ff_activation\", [\"gelu\", \"silu\"]),\n            \"head\": trial.suggest_categorical(\"head_activation\", [\"relu\", \"gelu\", \"silu\"])\n        }\n        batch_size    = trial.suggest_categorical(\"batch_size\", [64, 128, 256])\n        lr            = trial.suggest_float(\"lr\", 1e-5, 1e-3, log=True)\n        weight_decay  = trial.suggest_float(\"weight_decay\", 1e-7, 1e-4, log=True)\n        max_epochs    = trial.suggest_int(\"epochs\", 45, 65)\n        specaug_cfg = {\n            \"max_time_width\": trial.suggest_int(\"specaug_time_width\", 3, 15, step=3),\n            \"max_feature_width\": trial.suggest_int(\"specaug_feature_width\", 2, 10, step=2),\n            \"prob\": trial.suggest_float(\"specaug_prob\", 0.1, 0.5)\n        }\n        focal_gamma = trial.suggest_float(\"focal_gamma\", 1.0, 3.0)\n        mixup_alpha = trial.suggest_float(\"mixup_alpha\", 0.0, 0.3, step=0.01)\n        gesture_margin = trial.suggest_float(\"gesture_logit_margin\", 0.2, 0.5)\n        neg_margin = trial.suggest_float(\"gesture_neg_margin\", 0.05, 0.25)\n        margin_weight = trial.suggest_float(\"margin_weight\", 0.1, 0.5)\n        penalty_weight = trial.suggest_float(\"penalty_weight\", 0.2, 0.5)\n        N = trial.suggest_int(\"conf_penalty_update_epoch\", 5, 10)\n        gesture_ratio = trial.suggest_float(\"gesture_ratio\", 0.8, 0.95)\n\n        fold_scores = []\n        fold_binary_f1s = []\n        fold_macro_f1s = []\n\n        for fold, (train_idx, val_idx) in enumerate(\n            stratified_split_with_nontarget(X_tune, y_tune, groups_tune, n_splits=3)\n        ):\n            model = CNNTwoBranchModel(\n                imu_dim=len(imu_cols),\n                tof_thm_dim=len(tof_cols+thm_cols),\n                imu_cnn_cfg=imu_cnn_cfg,\n                tof_thm_cnn_cfg=tof_thm_cnn_cfg,\n                noise_dense_branch_cfg=noise_dense_branch_cfg,\n                lstm_cfg=lstm_cfg,\n                gru_cfg=gru_cfg,\n                trans_cfg=trans_cfg,\n                gesture_head_cfg=gesture_head_cfg,\n                binary_head_cfg=binary_head_cfg,\n                activation_cfg=activation_cfg,\n                num_classes=num_classes\n                ).to(device)\n\n            optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n            stopper = EarlyStopper(patience=8, min_delta=1e-4)\n            best_fold_score = -float(\"inf\")\n\n            train_X_gpu = X_tune[train_idx].to(device)\n            train_y_gpu = y_tune[train_idx].to(device)\n            val_X_gpu   = X_tune[val_idx].to(device)\n            val_y_gpu   = y_tune[val_idx].to(device)\n\n            train_loader = create_loader(train_X_gpu, train_y_gpu, batch_size)\n            val_loader   = create_loader(val_X_gpu, val_y_gpu, batch_size, shuffle=False)\n            train_gesture_labels = train_y_gpu[train_y_gpu != -1].cpu().numpy()\n            class_weights = compute_class_weight('balanced', classes=np.arange(num_classes), y=train_gesture_labels)\n            class_weights_tensor = torch.tensor(class_weights, dtype=torch.float32).to(device)\n            gesture_criterion = FocalLoss(gamma=focal_gamma, weight=class_weights_tensor)\n            binary_criterion = nn.BCEWithLogitsLoss()\n            global_step_base = fold * max_epochs\n            specaugment = SpecAugment(**specaug_cfg).to(device)\n            conf_penalty = None\n            for epoch in range(max_epochs):\n                model.train()\n                for xb, yb in train_loader:\n                    xb_full, yb_full = xb, yb\n                    mask = yb != -1\n                    xb_gesture, yb_gesture = xb[mask], yb[mask]\n                    xb_gesture = specaugment(xb_gesture)\n\n                    inputs, targets_a, targets_b, lam = mixup_data(xb_gesture, yb_gesture, alpha=mixup_alpha)\n                    optimizer.zero_grad()\n                    gesture_logits, binary_logits = model(xb_full)\n                    def mixup_criterion(criterion, pred, y_a, y_b, lam):\n                        mask_a = y_a != -1\n                        mask_b = y_b != -1\n                        mask = mask_a & mask_b\n                        if mask.sum() == 0:\n                            return torch.tensor(0.0, device=pred.device)\n                        return lam * criterion(pred[mask], y_a[mask]) + (1 - lam) * criterion(pred[mask], y_b[mask])\n\n                    def dual_margin_loss(logits, targets, pos_margin=0.3, neg_margin=0.1):\n                        B, C = logits.shape\n                        true_logits = logits.gather(1, targets.unsqueeze(1)).squeeze(1)  # [B]\n\n                        one_hot = F.one_hot(targets, num_classes=C).bool()\n                        logits_masked = logits.masked_fill(one_hot, float('-inf'))\n\n                        # Inter-class margin: max other\n                        max_other_logits = logits_masked.max(dim=1).values\n                        inter_margin_loss = F.relu(pos_margin - (true_logits - max_other_logits))\n\n                        # Intra-class spread: mean of other logits\n                        mean_other_logits = logits_masked.masked_fill(torch.isnan(logits_masked), 0).mean(dim=1)\n                        intra_margin_loss = F.relu(neg_margin - (true_logits - mean_other_logits))\n\n                        return inter_margin_loss.mean() + 0.05 * intra_margin_loss.mean()\n\n\n                    ce_loss = mixup_criterion(gesture_criterion, gesture_logits[mask], targets_a, targets_b, lam)\n                    margin_loss = dual_margin_loss(gesture_logits[mask], yb_gesture, pos_margin=gesture_margin, neg_margin=neg_margin)\n                    if conf_penalty is None:\n                        penalty = torch.tensor(0.0, device=device)\n                    else:\n                        gesture_probs = torch.softmax(gesture_logits[mask], dim=1)\n                        penalty = (conf_penalty[yb_gesture] * gesture_probs).sum(dim=1).mean()\n                    gesture_loss = ce_loss + margin_weight * margin_loss + penalty_weight * penalty\n\n                    binary_labels = (yb_full != -1).float().unsqueeze(1)\n                    binary_loss = binary_criterion(binary_logits, binary_labels)\n                    gesture_weight = gesture_ratio\n                    binary_weight = 1 - gesture_ratio\n                    loss = gesture_weight * gesture_loss + binary_weight * binary_loss + penalty_weight * penalty\n\n                    loss.backward()\n                    optimizer.step()\n\n                model.eval()\n                all_gesture_probs = []\n                all_binary_probs = []\n                all_labels = []\n\n                with torch.no_grad():\n                    for xb, yb in val_loader:\n                        gesture_logits, binary_logits = model(xb)\n\n                        gesture_probs = torch.softmax(gesture_logits, dim=1)\n                        binary_probs = torch.sigmoid(binary_logits).squeeze(-1)\n\n                        all_gesture_probs.append(gesture_probs.cpu().numpy())\n                        all_binary_probs.append(binary_probs.cpu().numpy())\n                        all_labels.append(yb.cpu().numpy())\n\n                all_gesture_probs = np.concatenate(all_gesture_probs, axis=0)\n                all_binary_probs = np.concatenate(all_binary_probs, axis=0)\n                all_labels = np.concatenate(all_labels, axis=0)\n\n                def optimize_binary_threshold(binary_probs, gesture_probs, labels):\n                    thresholds = np.linspace(0.05, 0.95, 50)\n                    best_score, best_thresh = -1, 0.5\n                    for th in thresholds:\n                        preds = np.full(len(labels), -1)\n                        mask = binary_probs >= th\n                        preds[mask] = np.argmax(gesture_probs[mask], axis=1)\n                        score, _, _ = competition_f1(labels, preds)\n                        if score > best_score:\n                            best_score, best_thresh = score, th\n                    return best_thresh\n\n                best_thresh = optimize_binary_threshold(all_binary_probs, all_gesture_probs, all_labels)\n                preds = np.full(len(all_labels), -1)\n                mask = all_binary_probs >= best_thresh\n                preds[mask] = np.argmax(all_gesture_probs[mask], axis=1)\n\n                score, binary_f1, macro_f1 = competition_f1(all_labels, preds)\n\n                warmup = 5\n                if epoch >= warmup and epoch % N == 0:\n                    cm = confusion_matrix(all_labels, preds, labels=np.arange(num_classes))\n                    cm = cm.astype(np.float32)\n                    conf_penalty = torch.tensor(cm / (cm.sum(axis=1, keepdims=True) + 1e-8), device=device)\n\n                trial.report(score, global_step_base + epoch)\n                if trial.should_prune():\n                    raise optuna.exceptions.TrialPruned()\n                if stopper.step(score, model):\n                    break\n                best_fold_score = max(best_fold_score, score)\n                model.train()\n\n            if stopper.best_state is not None:\n                model.load_state_dict(stopper.best_state)\n            fold_scores.append(score)\n            fold_binary_f1s.append(binary_f1)\n            fold_macro_f1s.append(macro_f1)\n\n\n        binary_str = \" | \".join([f\"B{i+1}={b:.4f}\" for i, b in enumerate(fold_binary_f1s)])\n        macro_str  = \" | \".join([f\"M{i+1}={m:.4f}\" for i, m in enumerate(fold_macro_f1s)])\n\n        mean_binary_f1 = np.mean(fold_binary_f1s)\n        mean_macro_f1  = np.mean(fold_macro_f1s)\n\n        log_msg = (f\"Trial {trial.number} | \"\n                   f\"{binary_str} || MeanB={mean_binary_f1:.4f} || \"\n                   f\"{macro_str} || MeanM={mean_macro_f1:.4f}\"\n                    )\n        optuna.logging.get_logger(\"optuna\").info(log_msg)\n        return -np.mean(fold_scores)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T19:32:22.607151Z","iopub.execute_input":"2025-09-02T19:32:22.607395Z","iopub.status.idle":"2025-09-02T19:32:22.642198Z","shell.execute_reply.started":"2025-09-02T19:32:22.607377Z","shell.execute_reply":"2025-09-02T19:32:22.641426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\nimport kaggle_evaluation.cmi_inference_server\n\n\nsave_dir = '/kaggle/input/cmi-checkpoint/'\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load resources once\nscaler = joblib.load(f\"{save_dir}/scaler.pkl\")\nlabel_encoder = joblib.load(f\"{save_dir}/label_encoder.pkl\")\nvalid_labels = set(label_encoder.classes_) | {\"non_target\"}\nnum_classes = len(label_encoder.classes_)\n\nwith open(os.path.join(save_dir, \"thresholds.json\"), \"r\") as f:\n    thresholds = json.load(f)\n    thresholds = {int(k): v for k, v in thresholds.items()}\n\nwith open(os.path.join(save_dir, \"best_params.json\"), \"r\") as f:\n    best_params = json.load(f)\n\n# Build model\nmodel = CNNTwoBranchModel(\n    imu_dim=len(imu_cols),\n    tof_thm_dim=len(tof_cols + thm_cols),\n    imu_cnn_cfg=best_params[\"imu_cnn_cfg\"],\n    tof_thm_cnn_cfg=best_params[\"tof_thm_cnn_cfg\"],\n    noise_dense_branch_cfg=best_params[\"noise_dense_branch_cfg\"],\n    lstm_cfg=best_params[\"lstm_cfg\"],\n    gru_cfg=best_params[\"gru_cfg\"],\n    trans_cfg=best_params[\"trans_cfg\"],\n    gesture_head_cfg=best_params[\"gesture_head_cfg\"],\n    binary_head_cfg=best_params[\"binary_head_cfg\"],\n    activation_cfg=best_params[\"activation_cfg\"],\n    num_classes=num_classes\n).to(device)\n\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    try:\n        use_full_model = False\n\n        df = sequence.to_pandas()\n        if df.empty:\n            return \"Text on phone\" \n\n        df = add_new_features(df)\n        df[imu_cols + thm_cols + tof_cols] = (\n            df[imu_cols + thm_cols + tof_cols]\n            .interpolate(method=\"linear\", limit_direction=\"both\", axis=0)\n            .fillna(0.0)\n        )\n        df[imu_cols + thm_cols + tof_cols] = scaler.transform(df[imu_cols + thm_cols + tof_cols])\n\n        df_group = df.copy()\n        df_group[imu_cols + thm_cols + tof_cols] = group_normalize(df_group, imu_cols + thm_cols + tof_cols)\n\n        global_tensor, _ = build_tensor(df, imu_cols + thm_cols + tof_cols, cap_len=120, mode=\"eval\")\n        group_tensor, _ = build_tensor(df_group, imu_cols + thm_cols + tof_cols, cap_len=120, mode=\"eval\")\n\n        x = torch.cat([global_tensor, group_tensor], dim=2).to(device)\n\n        gesture_outputs = []\n        binary_outputs = []\n\n        for fold in range(5):\n            model.load_state_dict(torch.load(os.path.join(save_dir, f\"fold{fold}_model.pt\"), map_location=device))\n            model.eval()\n            with torch.no_grad():\n                gesture_logits, binary_logits = model(x)\n                gesture_probs = torch.softmax(gesture_logits, dim=1).cpu().numpy().squeeze()\n                binary_prob = torch.sigmoid(binary_logits).cpu().numpy().squeeze()\n                gesture_outputs.append(gesture_probs)\n                binary_outputs.append(binary_prob)\n\n        avg_gesture_probs = np.mean(gesture_outputs, axis=0)\n        avg_binary_prob = np.mean(binary_outputs)\n        avg_threshold = np.mean([thresholds[i] for i in range(5)])\n\n        if avg_binary_prob < avg_threshold:\n            return \"Text on phone\" \n\n        pred_class_idx = avg_gesture_probs.argmax()\n        pred_class = label_encoder.inverse_transform([pred_class_idx])[0]\n\n        return str(pred_class) if pred_class in valid_labels else \"non_target\"\n\n    except Exception as e:\n        print(f\"Exception in predict(): {e}\")\n        import traceback\n        traceback.print_exc()\n        return \"Text on phone\"\n\n\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T19:37:15.521274Z","iopub.status.idle":"2025-09-02T19:37:15.521587Z","shell.execute_reply.started":"2025-09-02T19:37:15.521427Z","shell.execute_reply":"2025-09-02T19:37:15.521442Z"}},"outputs":[],"execution_count":null}]}