{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":3712905,"sourceType":"datasetVersion","datasetId":2220792},{"sourceId":12551241,"sourceType":"datasetVersion","datasetId":7924751},{"sourceId":12725524,"sourceType":"datasetVersion","datasetId":8043349},{"sourceId":12744889,"sourceType":"datasetVersion","datasetId":8056620},{"sourceId":12748572,"sourceType":"datasetVersion","datasetId":8058903},{"sourceId":12764241,"sourceType":"datasetVersion","datasetId":8069084},{"sourceId":12765191,"sourceType":"datasetVersion","datasetId":8069697},{"sourceId":12769692,"sourceType":"datasetVersion","datasetId":8072607},{"sourceId":12787148,"sourceType":"datasetVersion","datasetId":8084252},{"sourceId":12787856,"sourceType":"datasetVersion","datasetId":8084889},{"sourceId":12787981,"sourceType":"datasetVersion","datasetId":8084983},{"sourceId":12836749,"sourceType":"datasetVersion","datasetId":8118645},{"sourceId":12836763,"sourceType":"datasetVersion","datasetId":8118655},{"sourceId":12837527,"sourceType":"datasetVersion","datasetId":8119152},{"sourceId":12838191,"sourceType":"datasetVersion","datasetId":8119600},{"sourceId":12845000,"sourceType":"datasetVersion","datasetId":8124163},{"sourceId":12874459,"sourceType":"datasetVersion","datasetId":8144291},{"sourceId":12875383,"sourceType":"datasetVersion","datasetId":7609865},{"sourceId":12885983,"sourceType":"datasetVersion","datasetId":8152667},{"sourceId":12886045,"sourceType":"datasetVersion","datasetId":8152710},{"sourceId":12927623,"sourceType":"datasetVersion","datasetId":8180255},{"sourceId":12927671,"sourceType":"datasetVersion","datasetId":8180286},{"sourceId":12930518,"sourceType":"datasetVersion","datasetId":8182326},{"sourceId":12930691,"sourceType":"datasetVersion","datasetId":8182456},{"sourceId":12930776,"sourceType":"datasetVersion","datasetId":8182507},{"sourceId":12936371,"sourceType":"datasetVersion","datasetId":8186056},{"sourceId":242954653,"sourceType":"kernelVersion"},{"sourceId":261069925,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install /kaggle/input/cmi-detect-behavior/polars-1.30.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n!pip install /kaggle/input/cmi-detect-behavior/scikit_learn-1.5.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:16.052068Z","iopub.execute_input":"2025-09-02T05:22:16.052294Z","iopub.status.idle":"2025-09-02T05:22:30.32085Z","shell.execute_reply.started":"2025-09-02T05:22:16.052267Z","shell.execute_reply":"2025-09-02T05:22:30.319892Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom cmi_2025_metric_copy_for_import import CompetitionMetric\n\nimport polars as pl\nfrom scipy.spatial.transform import Rotation as R\nimport math\nimport sys\nsys.path.append('/kaggle/input/cmi-utils-top1/')\nfrom cmi_utils import *\n\ndevice = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:30.321952Z","iopub.execute_input":"2025-09-02T05:22:30.322182Z","iopub.status.idle":"2025-09-02T05:22:36.618478Z","shell.execute_reply.started":"2025-09-02T05:22:30.322159Z","shell.execute_reply":"2025-09-02T05:22:36.617852Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Configuration","metadata":{}},{"cell_type":"code","source":"RAW_DIR = Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")\n\nall_features_lstm_dir = Path(\"/kaggle/input/ensemble-55/exp_334_comb\")\nall_features_lstm_dir2 = Path(\"/kaggle/input/ensemble-55/exp_337_comb\") \nall_features_attention_dir = Path(\"/kaggle/input/ensemble-55/exp_338_comb\") \nall_features_cnn_dir = Path(\"/kaggle/input/ensemble-55/exp_339_comb\")\n\nall_features_lstm_dir_old = Path(\"/kaggle/input/ensemble-32/exp_317_comb\")\nall_features_lstm_dir2_old = Path(\"/kaggle/input/ensemble-32/exp_289\") \nall_features_attention_dir_old = Path(\"/kaggle/input/ensemble-32/exp_252_comb\") \nall_features_cnn_dir_old = Path(\"/kaggle/input/ensemble-32/exp_315_comb\")\n\nimu_features_lstm_dir = Path(\"/kaggle/input/ensemble-55/exp_230\") \nimu_features_lstm_dir2 = Path(\"/kaggle/input/ensemble-55/exp_272_comb\") \nimu_features_attention_dir = Path(\"/kaggle/input/ensemble-55/exp_254_comb\")\nimu_features_cnn_dir = Path(\"/kaggle/input/ensemble-55/exp_267\") \n\nmodel_all_features_path = \"three_branch_mixup_best_f1_fold\"\nmodel_imu_features_path = \"one_branch_mixup_best_f1_fold\"\n\nACC_COLS = [\"acc_x\", \"acc_y\", \"acc_z\"]\nROT_COLS = [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n\n# NEW\nimport timm\nimu_specs_dir1 = Path(\"/kaggle/input/cmi-specs-models/exp_343\")\nimu_specs_dir2 = Path(\"/kaggle/input/cmi-specs-models/exp_342\")\n# /NEW","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:36.62002Z","iopub.execute_input":"2025-09-02T05:22:36.6204Z","iopub.status.idle":"2025-09-02T05:22:44.269766Z","shell.execute_reply.started":"2025-09-02T05:22:36.620381Z","shell.execute_reply":"2025-09-02T05:22:44.268922Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Data Helpers","metadata":{}},{"cell_type":"code","source":"def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n\ndef compute_world_acceleration(acc: np.ndarray, rot: np.ndarray) -> np.ndarray:\n \"\"\"\n Convert acceleration from device coordinates to world coordinates\n \n This is the key innovation: normalizing for device orientation\n \n Args:\n acc: acceleration in device coordinates, shape (time_steps, 3) [x, y, z]\n rot: rotation quaternion, shape (time_steps, 4) [w, x, y, z] (normalized)\n \n Returns:\n acc_world: acceleration in world coordinates, shape (time_steps, 3)\n \n Why this matters:\n - Device acceleration depends on how the watch is oriented on the wrist\n - World acceleration is independent of device orientation\n - This helps the model focus on actual hand motion rather than wrist rotation\n \"\"\"\n try:\n # Convert quaternion format from [w, x, y, z] to [x, y, z, w] for scipy\n rot_scipy = rot[:, [1, 2, 3, 0]]\n \n # Verify quaternions are valid (non-zero norm)\n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n # Create rotation object and apply transformation\n r = R.from_quat(rot_scipy)\n acc_world = r.apply(acc)\n \n except Exception:\n # Fallback to original acceleration if transformation fails\n print(\"Warning: World coordinate transformation failed, using device coordinates\")\n acc_world = acc.copy()\n \n return acc_world","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.270608Z","iopub.execute_input":"2025-09-02T05:22:44.271051Z","iopub.status.idle":"2025-09-02T05:22:44.280317Z","shell.execute_reply.started":"2025-09-02T05:22:44.27103Z","shell.execute_reply":"2025-09-02T05:22:44.279572Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200): # Assuming 200Hz sampling rate\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n\n for i in range(num_samples - 1):\n q_t = quat_values[i]\n q_t_plus_dt = quat_values[i+1]\n\n if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n continue\n\n try:\n rot_t = R.from_quat(q_t)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n\n # Calculate the relative rotation\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n # Convert delta rotation to angular velocity vector\n # The rotation vector (Euler axis * angle) scaled by 1/dt\n # is a good approximation for small delta_rot\n angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n except ValueError:\n # If quaternion is invalid, angular velocity remains zero\n pass\n \n return angular_vel\n\ndef calculate_angular_distance(rot_data):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n\n for i in range(num_samples - 1):\n q1 = quat_values[i]\n q2 = quat_values[i+1]\n\n if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n angular_dist[i] = 0 # Или np.nan, в зависимости от желаемого поведения\n continue\n try:\n # Преобразование кватернионов в объекты Rotation\n r1 = R.from_quat(q1)\n r2 = R.from_quat(q2)\n\n # Вычисление углового расстояния: 2 * arccos(|real(p * q*)|)\n # где p* - сопряженный кватернион q\n # В scipy.spatial.transform.Rotation, r1.inv() * r2 дает относительное вращение.\n # Угол этого относительного вращения - это и есть угловое расстояние.\n relative_rotation = r1.inv() * r2\n \n # Угол rotation vector соответствует угловому расстоянию\n # Норма rotation vector - это угол в радианах\n angle = np.linalg.norm(relative_rotation.as_rotvec())\n angular_dist[i] = angle\n except ValueError:\n angular_dist[i] = 0 # В случае недействительных кватернионов\n pass\n \n return angular_dist","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.281403Z","iopub.execute_input":"2025-09-02T05:22:44.281725Z","iopub.status.idle":"2025-09-02T05:22:44.3051Z","shell.execute_reply.started":"2025-09-02T05:22:44.281706Z","shell.execute_reply":"2025-09-02T05:22:44.304566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def remove_gravity_from_acc(acc_data, rot_data):\n\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = acc_data\n\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = rot_data\n\n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n \n gravity_world = np.array([0, 0, 9.81])\n\n for i in range(num_samples):\n if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n linear_accel[i, :] = acc_values[i, :] \n continue\n\n try:\n rotation = R.from_quat(quat_values[i])\n gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n except ValueError:\n linear_accel[i, :] = acc_values[i, :]\n \n return linear_accel","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.305861Z","iopub.execute_input":"2025-09-02T05:22:44.306151Z","iopub.status.idle":"2025-09-02T05:22:44.321347Z","shell.execute_reply.started":"2025-09-02T05:22:44.306124Z","shell.execute_reply":"2025-09-02T05:22:44.320744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def add_features(df: pd.DataFrame) -> pd.DataFrame:\n\n \"\"\"\n Add world-frame acceleration features to the dataframe.\n\n This function computes acceleration in world coordinates using quaternion orientation,\n and appends these as new columns to the dataframe.\n\n Args:\n df (pd.DataFrame): Input dataframe containing sensor data and metadata.\n\n Returns:\n pd.DataFrame: Dataframe with added world-frame acceleration columns.\n \"\"\"\n\n meta_cols = [\n \"sequence_id\"\n ]\n feature_cols_all = [c for c in df.columns if c not in meta_cols]\n other_cols = [c for c in feature_cols_all if (c.startswith('thm_') or c.startswith('tof_'))]\n \n # Add features https://www.kaggle.com/code/rktqwe/lb-0-77-linear-accel-tf-bilstm-gru-attention\n df['acc_mag'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n df['acc_mag_jerk'] = df['acc_mag'].diff().fillna(0)\n\n # Jerk magnitude (rate of change of acceleration)\n df['jerk_x'] = np.gradient(df['acc_x'])\n df['jerk_y'] = np.gradient(df['acc_y']) \n df['jerk_z'] = np.gradient(df['acc_z'])\n df['jerk_magnitude'] = np.sqrt(df['jerk_x']**2 + df['jerk_y']**2 + df['jerk_z']**2)\n \n # Correlation between axes (rolling correlation)\n cor_list = []\n window = 20\n for _, group in df.groupby('sequence_id'):\n group['acc_xy_corr'] = group['acc_x'].rolling(window).corr(group['acc_y']).fillna(0)\n group['acc_xz_corr'] = group['acc_x'].rolling(window).corr(group['acc_z']).fillna(0)\n group['acc_yz_corr'] = group['acc_y'].rolling(window).corr(group['acc_z']).fillna(0)\n cor_list.append(group[['acc_xy_corr', 'acc_xz_corr', 'acc_yz_corr']].to_numpy())\n cor_list = np.concatenate(cor_list, axis = 0)\n df[['acc_xy_corr', 'acc_xz_corr', 'acc_yz_corr']] = cor_list\n\n df['rot_angle'] = 2 * np.arccos(df['rot_w'].clip(-1, 1))\n df['rot_angle_vel'] = df['rot_angle'].diff().fillna(0) \n\n print(\"Calculating angular velocity from quaternion derivatives...\")\n angular_vel_list = []\n for _, group in df.groupby('sequence_id'):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(angular_vel_group)\n angular_vel_list = np.concatenate(angular_vel_list, axis = 0)\n df[['angular_vel_x', 'angular_vel_y', 'angular_vel_z']] = angular_vel_list\n \n df['angular_vel_magnitude'] = np.sqrt(\n df['angular_vel_x']**2 + df['angular_vel_y']**2 + df['angular_vel_x']**2\n )\n\n print(\"Calculating angular distance between successive quaternions...\")\n angular_distance_list = []\n for _, group in df.groupby('sequence_id'):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(angular_dist_group)\n angular_distance_list = np.concatenate(angular_distance_list, axis = 0)\n df['angular_distance'] = angular_distance_list\n \n ############################################\n\n ACC_COLS2 = [\"acc_x2\", \"acc_y2\", \"acc_z2\"]\n\n linear_accel_list = []\n for _, group in df.groupby('sequence_id'):\n acc_data_group = group[ACC_COLS]\n rot_data_group = group[ROT_COLS]\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(linear_accel_group)\n linear_accel_list = np.concatenate(linear_accel_list , axis = 0)\n df[ACC_COLS2] = linear_accel_list\n\n # Add features https://www.kaggle.com/code/rktqwe/lb-0-77-linear-accel-tf-bilstm-gru-attention\n df['acc_mag2'] = np.sqrt(df['acc_x2']**2 + df['acc_y2']**2 + df['acc_z2']**2)\n df['acc_mag_jerk2'] = df['acc_mag2'].diff().fillna(0)\n\n # Jerk magnitude (rate of change of acceleration)\n df['jerk_x2'] = np.gradient(df['acc_x2'])\n df['jerk_y2'] = np.gradient(df['acc_y2']) \n df['jerk_z2'] = np.gradient(df['acc_z2'])\n df['jerk_magnitude2'] = np.sqrt(df['jerk_x2']**2 + df['jerk_y2']**2 + df['jerk_z2']**2)\n \n # Correlation between axes (rolling correlation)\n cor_list = []\n window = 20\n for _, group in df.groupby('sequence_id'):\n group['acc_xy_corr2'] = group['acc_x2'].rolling(window).corr(group['acc_y2']).fillna(0)\n group['acc_xz_corr2'] = group['acc_x2'].rolling(window).corr(group['acc_z2']).fillna(0)\n group['acc_yz_corr2'] = group['acc_y2'].rolling(window).corr(group['acc_z2']).fillna(0)\n cor_list.append(group[['acc_xy_corr2', 'acc_xz_corr2', 'acc_yz_corr2']].to_numpy())\n cor_list = np.concatenate(cor_list, axis = 0)\n df[['acc_xy_corr2', 'acc_xz_corr2', 'acc_yz_corr2']] = cor_list\n\n df = df[meta_cols \n + ACC_COLS + ['acc_mag', 'acc_mag_jerk'] \n + ['jerk_x', 'jerk_y', 'jerk_z', 'jerk_magnitude']\n + [\"acc_xy_corr\", \"acc_xz_corr\", \"acc_yz_corr\"]\n + ROT_COLS + ['rot_angle', 'rot_angle_vel'] \n + ['angular_vel_x', 'angular_vel_y', 'angular_vel_z'] \n + ['angular_vel_magnitude', 'angular_distance'] \n + ACC_COLS2 + ['acc_mag2', 'acc_mag_jerk2'] \n + ['jerk_x2', 'jerk_y2', 'jerk_z2', 'jerk_magnitude2']\n + [\"acc_xy_corr2\", \"acc_xz_corr2\", \"acc_yz_corr2\"] + other_cols]\n # x = x.ffill().bfill().fillna(0).to_numpy(dtype = dtype)\n\n df.replace(-np.inf, -1, inplace=True)\n df.replace(np.inf, 1, inplace=True)\n\n return df\n\n\ndef pad_sequences(\n sequences: list[np.ndarray], \n maxlen: int, \n padding: str = \"pre\", \n truncating: str = \"pre\", \n dtype: str = \"float32\"\n) -> np.ndarray:\n \n \"\"\"\n Pad sequences to the same length.\n\n Parameters:\n -----------\n sequences : list of numpy.ndarray\n List of sequences (each sequence is a numpy array) to pad\n maxlen : int\n Maximum length of all sequences. Sequences longer than maxlen will be truncated\n padding : str, optional (default=\"pre\")\n 'pre' or 'post', pad either before or after each sequence\n truncating : str, optional (default=\"pre\")\n 'pre' or 'post', remove values from sequences larger than maxlen either at the\n beginning or at the end\n dtype : str, optional (default=\"float32\")\n Type of the output array\n\n Returns:\n --------\n numpy.ndarray\n Padded sequences array of shape (len(sequences), maxlen, ...)\n \"\"\"\n \n if padding not in [\"pre\", \"post\"]: \n raise NotImplementedError(\"Invalid padding\")\n if truncating not in [\"pre\", \"post\"]: \n raise NotImplementedError(\"Invalid truncating\")\n \n n_samples = len(sequences)\n if maxlen is None:\n maxlen = max(len(s) for s in sequences)\n \n # Sample shape from first non empty sequence\n # If no non-empty sequences, return array of shape (0, maxlen)\n sample_shape = tuple()\n for s in sequences:\n if len(s) > 0:\n sample_shape = np.asarray(s).shape[1:]\n break\n \n x = np.zeros((n_samples, maxlen) + sample_shape, dtype = dtype)\n \n for idx, s in enumerate(sequences):\n if len(s) == 0:\n continue\n \n if truncating == \"pre\":\n s = s[-maxlen:] \n elif truncating == \"post\":\n s[:maxlen]\n\n trunc = s.shape[0]\n \n if padding == \"pre\":\n x[idx, -trunc:] = s\n elif padding == \"post\":\n x[idx, :trunc] = s\n \n return x\n\ndef to_categorical(y, num_classes = None):\n y = np.array(y, dtype = \"int\")\n if not num_classes:\n num_classes = np.max(y) + 1\n return np.eye(num_classes)[y]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.322201Z","iopub.execute_input":"2025-09-02T05:22:44.322448Z","iopub.status.idle":"2025-09-02T05:22:44.342839Z","shell.execute_reply.started":"2025-09-02T05:22:44.322427Z","shell.execute_reply":"2025-09-02T05:22:44.342302Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mirror_quaternion(quat):\n \"\"\"\n Mirror a single quaternion through the YZ plane.\n\n Args:\n quat (array-like of shape (4,)): [w, x, y, z]\n\n Returns:\n mirrored (np.ndarray of shape (4,)): mirrored quaternion [w, x, y, z]\n \"\"\"\n P = np.diag([-1, 1, 1]) # reflection through YZ\n rot = R.from_quat([quat[1], quat[2], quat[3], quat[0]]) # SciPy uses [x, y, z, w]\n R_mat = rot.as_matrix()\n R_flipped = P @ R_mat @ P\n flipped = R.from_matrix(R_flipped).as_quat()\n return np.array([flipped[3], flipped[0], flipped[1], flipped[2]]) # back to [w, x, y, z]\n\n\ndef mirror_data(data):\n \"\"\"\n Mirror left-handed samples to match right-handed frame.\n\n Args:\n data (np.ndarray of shape (N, 7)): sensor data\n \n Prints:\n A new array with mirrored left-handed samples.\n\n \"\"\"\n mirrored = data.copy()\n \n for i in range(data.shape[0]):\n acc = mirrored[i, :3]\n mirrored[i, :3] = np.array([-acc[0], acc[1], acc[2]])\n\n quat = mirrored[i, 3:] # [w, x, y, z]\n mirrored[i, 3:] = mirror_quaternion(quat)\n\n return mirrored","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.343587Z","iopub.execute_input":"2025-09-02T05:22:44.343842Z","iopub.status.idle":"2025-09-02T05:22:44.360255Z","shell.execute_reply.started":"2025-09-02T05:22:44.343824Z","shell.execute_reply":"2025-09-02T05:22:44.359603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_left_handed_pd(df: pd.DataFrame, demo: pd.DataFrame) -> pd.DataFrame:\n \"\"\"Note this may sort your dataframe by 'row_id', \n Also removes nans from rotation, and probably converts to unit\"\"\"\n def handle_missing(rot_data: np.ndarray) -> np.ndarray:\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n\n import polars as pl\n pl_df = pl.DataFrame(df)\n pl_demo = pl.DataFrame(demo)\n r_subjs = pl_demo.filter(pl.col(\"handedness\") == 1)[\"subject\"].to_list()\n l_subjs = pl_demo.filter(pl.col(\"handedness\") == 0)[\"subject\"].to_list()\n \n r_tr = pl_df.filter(pl.col(\"subject\").is_in(r_subjs))\n l_tr = pl_df.filter(pl.col(\"subject\").is_in(l_subjs))\n\n if l_tr.shape[0] == 0:\n return r_tr.to_pandas()\n\n rot_cleaned = handle_missing(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n print(\"yes\")\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] # maybe who knows but you could give it a try \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n\n if r_tr.shape[0] == 0:\n return l_tr.to_pandas()\n else:\n pl_df = pl.concat([r_tr, l_tr])\n pl_df = pl_df.sort(by=\"row_id\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.362958Z","iopub.execute_input":"2025-09-02T05:22:44.363146Z","iopub.status.idle":"2025-09-02T05:22:44.380911Z","shell.execute_reply.started":"2025-09-02T05:22:44.363123Z","shell.execute_reply":"2025-09-02T05:22:44.380207Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. All features models","metadata":{}},{"cell_type":"markdown","source":"## 3.1. All features - LSTM","metadata":{}},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n \nclass CrossAttention(nn.Module):\n \"\"\"\n Cross-attention module for multi-branch feature fusion.\n Allows each branch to attend to features from other branches.\n \"\"\"\n def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n super(CrossAttention, self).__init__()\n self.feature_dim = feature_dim\n self.num_heads = num_heads\n self.head_dim = feature_dim // num_heads\n \n assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n \n # Linear projections for queries, keys, and values for each branch\n self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n \n # Output projection\n self.out_linear = nn.Linear(feature_dim, feature_dim)\n self.dropout = nn.Dropout(dropout)\n \n # Layer normalization for residual connection\n self.layer_norm = nn.LayerNorm(feature_dim)\n \n def forward(self, query_branch, key_value_branches):\n \"\"\"\n Args:\n query_branch: (B, T, C) - the branch that will be updated\n key_value_branches: List of (B, T, C) tensors - branches to attend to\n Returns:\n Updated query_branch with cross-attention: (B, T, C)\n \"\"\"\n B, T, C = query_branch.shape\n \n # Create queries from the query branch\n Q = self.q_linear(query_branch) # (B, T, C)\n \n # Concatenate all key-value branches for attention\n all_kv = torch.stack(key_value_branches, dim=1) # (B, num_branches, T, C)\n num_branches = all_kv.shape[1]\n \n # Reshape for multi-head attention\n all_kv = all_kv.reshape(B, num_branches * T, C) # (B, num_branches*T, C)\n \n K = self.k_linear(all_kv) # (B, num_branches*T, C)\n V = self.v_linear(all_kv) # (B, num_branches*T, C)\n \n # Reshape for multi-head attention\n Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, T, head_dim)\n K = K.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n V = V.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n \n # Scaled dot-product attention\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # (B, num_heads, T, num_branches*T)\n attn_weights = F.softmax(scores, dim=-1)\n attn_weights = self.dropout(attn_weights)\n \n # Apply attention to values\n attn_output = torch.matmul(attn_weights, V) # (B, num_heads, T, head_dim)\n \n # Concatenate heads and put through final linear layer\n attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)\n output = self.out_linear(attn_output)\n \n # Residual connection and layer normalization\n output = self.layer_norm(query_branch + output)\n \n return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n \"\"\"\n Cross-attention fusion module for three IMU branches.\n Each branch attends to the other two branches.\n \"\"\"\n def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n super(IMUCrossAttentionFusion, self).__init__()\n \n # Cross-attention modules for each branch\n self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn4 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn5 = CrossAttention(feature_dim, num_heads, dropout)\n \n def forward(self, imu1, imu2, imu3, thm, tof):\n \"\"\"\n Args:\n imu1, imu2, imu3, thm, tof: (B, T, C) tensors from each branch\n Returns:\n Tuple of enhanced features: (enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof)\n \"\"\"\n # Each branch attends to the other two branches\n enhanced_imu1 = self.cross_attn1(imu1, [imu2, imu3, thm, tof])\n enhanced_imu2 = self.cross_attn2(imu2, [imu1, imu3, thm, tof])\n enhanced_imu3 = self.cross_attn3(imu3, [imu1, imu2, thm, tof])\n enhanced_thm = self.cross_attn4(thm, [imu1, imu2, imu3, tof])\n enhanced_tof = self.cross_attn5(tof, [imu1, imu2, imu3, thm])\n \n return enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof\n\n\nclass ImuModelLSTM(nn.Module):\n def __init__(self, imu_dim):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branchs\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n \n return imu1, imu2, imu3\n\nclass ThreeBranchModelLSTM(nn.Module):\n def __init__(self, imu_dim, thm_dim, tof_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n self.thm_dim = thm_dim\n self.tof_dim = tof_dim\n \n # IMU branch\n self.imu_branch = ImuModelLSTM(self.imu_dim)\n\n # THM branch\n self.thm_branch = nn.Sequential(\n ResidualSEBlock(thm_dim, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n # TOF branch\n self.tof_branch = nn.Sequential(\n ResidualSEBlock(tof_dim, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n self.cross_attention_fusion = IMUCrossAttentionFusion(\n feature_dim=256, num_heads=8, dropout=0.1\n )\n \n # BiLSTM\n self.bilstm = nn.LSTM(256*5, 512, num_layers=1, bidirectional=True, batch_first=True)\n self.dropout = nn.Dropout(0.4)\n \n # Attention\n self.attention = AttentionLayer(1024)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(256*4, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n # Split input\n imu = x[:, :, :self.imu_dim]\n thm = x[:, :, self.imu_dim:(self.imu_dim+self.thm_dim)]\n tof = x[:, :, (self.imu_dim+self.thm_dim):]\n # Handle missing values (-1) by replacing with 500\n tof = torch.where(tof == -1, torch.full_like(tof, 500), tof)\n # Normalize to [0, 1] range\n tof = tof / 500.0\n \n # Process branches\n imu1, imu2, imu3 = self.imu_branch(imu)\n thm = self.thm_branch(thm.transpose(1, 2)).transpose(1, 2)\n tof = self.tof_branch(tof.transpose(1, 2)).transpose(1, 2)\n\n imu1, imu2, imu3, thm, tof = self.cross_attention_fusion(imu1, imu2, imu3, thm, tof)\n\n merged = torch.cat((imu1, imu2, imu3, thm, tof), dim=2)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n lstm_out = self.dropout(lstm_out)\n \n # Attention\n attended = self.attention(lstm_out)\n \n # Dense layers\n out = self.fc_layers(attended)\n return out ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.381571Z","iopub.execute_input":"2025-09-02T05:22:44.381797Z","iopub.status.idle":"2025-09-02T05:22:44.409521Z","shell.execute_reply.started":"2025-09-02T05:22:44.381782Z","shell.execute_reply":"2025-09-02T05:22:44.408902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_lstm_dir / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_lstm_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_lstm = []\nfor fold in range(5):\n model = ThreeBranchModelLSTM(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_lstm_dir / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_lstm.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:44.410224Z","iopub.execute_input":"2025-09-02T05:22:44.410493Z","iopub.status.idle":"2025-09-02T05:22:48.079345Z","shell.execute_reply.started":"2025-09-02T05:22:44.410477Z","shell.execute_reply":"2025-09-02T05:22:48.078504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_lstm_dir_old / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_lstm_dir_old / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_lstm_old = []\nfor fold in range(5):\n model = ThreeBranchModelLSTM(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_lstm_dir_old / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_lstm_old.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:48.08018Z","iopub.execute_input":"2025-09-02T05:22:48.080518Z","iopub.status.idle":"2025-09-02T05:22:50.785323Z","shell.execute_reply.started":"2025-09-02T05:22:48.080495Z","shell.execute_reply":"2025-09-02T05:22:50.784519Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n \nclass CrossAttention(nn.Module):\n \"\"\"\n Cross-attention module for multi-branch feature fusion.\n Allows each branch to attend to features from other branches.\n \"\"\"\n def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n super(CrossAttention, self).__init__()\n self.feature_dim = feature_dim\n self.num_heads = num_heads\n self.head_dim = feature_dim // num_heads\n \n assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n \n # Linear projections for queries, keys, and values for each branch\n self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n \n # Output projection\n self.out_linear = nn.Linear(feature_dim, feature_dim)\n self.dropout = nn.Dropout(dropout)\n \n # Layer normalization for residual connection\n self.layer_norm = nn.LayerNorm(feature_dim)\n \n def forward(self, query_branch, key_value_branches):\n \"\"\"\n Args:\n query_branch: (B, T, C) - the branch that will be updated\n key_value_branches: List of (B, T, C) tensors - branches to attend to\n Returns:\n Updated query_branch with cross-attention: (B, T, C)\n \"\"\"\n B, T, C = query_branch.shape\n \n # Create queries from the query branch\n Q = self.q_linear(query_branch) # (B, T, C)\n \n # Concatenate all key-value branches for attention\n all_kv = torch.stack(key_value_branches, dim=1) # (B, num_branches, T, C)\n num_branches = all_kv.shape[1]\n \n # Reshape for multi-head attention\n all_kv = all_kv.reshape(B, num_branches * T, C) # (B, num_branches*T, C)\n \n K = self.k_linear(all_kv) # (B, num_branches*T, C)\n V = self.v_linear(all_kv) # (B, num_branches*T, C)\n \n # Reshape for multi-head attention\n Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, T, head_dim)\n K = K.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n V = V.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n \n # Scaled dot-product attention\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # (B, num_heads, T, num_branches*T)\n attn_weights = F.softmax(scores, dim=-1)\n attn_weights = self.dropout(attn_weights)\n \n # Apply attention to values\n attn_output = torch.matmul(attn_weights, V) # (B, num_heads, T, head_dim)\n \n # Concatenate heads and put through final linear layer\n attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)\n output = self.out_linear(attn_output)\n \n # Residual connection and layer normalization\n output = self.layer_norm(query_branch + output)\n \n return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n \"\"\"\n Cross-attention fusion module for three IMU branches.\n Each branch attends to the other two branches.\n \"\"\"\n def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n super(IMUCrossAttentionFusion, self).__init__()\n \n # Cross-attention modules for each branch\n self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn4 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn5 = CrossAttention(feature_dim, num_heads, dropout)\n \n def forward(self, imu1, imu2, imu3, thm, tof):\n \"\"\"\n Args:\n imu1, imu2, imu3, thm, tof: (B, T, C) tensors from each branch\n Returns:\n Tuple of enhanced features: (enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof)\n \"\"\"\n # Each branch attends to the other two branches\n enhanced_imu1 = self.cross_attn1(imu1, [imu2, imu3, thm, tof])\n enhanced_imu2 = self.cross_attn2(imu2, [imu1, imu3, thm, tof])\n enhanced_imu3 = self.cross_attn3(imu3, [imu1, imu2, thm, tof])\n enhanced_thm = self.cross_attn3(thm, [imu1, imu2, imu3, tof])\n enhanced_tof = self.cross_attn3(tof, [imu1, imu2, imu3, thm])\n \n return enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof\n\n\nclass ImuModelLSTM2(nn.Module):\n def __init__(self, imu_dim):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branchs\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n \n return imu1, imu2, imu3\n\nclass ThreeBranchModelLSTM2(nn.Module):\n def __init__(self, imu_dim, thm_dim, tof_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n self.thm_dim = thm_dim\n self.tof_dim = tof_dim\n \n # IMU branch\n self.imu_branch = ImuModelLSTM2(self.imu_dim)\n\n # THM branch\n self.thm_branch = nn.Sequential(\n ResidualSEBlock(thm_dim, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n # TOF branch\n self.tof_branch = nn.Sequential(\n ResidualSEBlock(tof_dim, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n self.cross_attention_fusion = IMUCrossAttentionFusion(\n feature_dim=256, num_heads=8, dropout=0.1\n )\n \n # BiLSTM\n self.bilstm = nn.LSTM(256*5, 512, num_layers=1, bidirectional=True, batch_first=True)\n self.dropout = nn.Dropout(0.4)\n \n # Attention\n self.attention = AttentionLayer(1024)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(256*4, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n # Split input\n imu = x[:, :, :self.imu_dim]\n thm = x[:, :, self.imu_dim:(self.imu_dim+self.thm_dim)]\n tof = x[:, :, (self.imu_dim+self.thm_dim):]\n # Handle missing values (-1) by replacing with 500\n tof = torch.where(tof == -1, torch.full_like(tof, 500), tof)\n # Normalize to [0, 1] range\n tof = tof / 500.0\n \n # Process branches\n imu1, imu2, imu3 = self.imu_branch(imu)\n thm = self.thm_branch(thm.transpose(1, 2)).transpose(1, 2)\n tof = self.tof_branch(tof.transpose(1, 2)).transpose(1, 2)\n\n imu1, imu2, imu3, thm, tof = self.cross_attention_fusion(imu1, imu2, imu3, thm, tof)\n\n merged = torch.cat((imu1, imu2, imu3, thm, tof), dim=2)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n lstm_out = self.dropout(lstm_out)\n \n # Attention\n attended = self.attention(lstm_out)\n \n # Dense layers\n out = self.fc_layers(attended)\n return out \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:50.786314Z","iopub.execute_input":"2025-09-02T05:22:50.786566Z","iopub.status.idle":"2025-09-02T05:22:50.814791Z","shell.execute_reply.started":"2025-09-02T05:22:50.786548Z","shell.execute_reply":"2025-09-02T05:22:50.814025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Use ThreeBranchModelLSTM for model with bug only\nfeature_cols = np.load(all_features_lstm_dir2 / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_lstm_dir2 / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_lstm2 = []\nfor fold in range(5):\n model = ThreeBranchModelLSTM(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_lstm_dir2 / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_lstm2.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:50.815738Z","iopub.execute_input":"2025-09-02T05:22:50.815978Z","iopub.status.idle":"2025-09-02T05:22:53.708917Z","shell.execute_reply.started":"2025-09-02T05:22:50.815955Z","shell.execute_reply":"2025-09-02T05:22:53.708103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_lstm_dir2_old / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_lstm_dir2_old / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_lstm2_old = []\nfor fold in range(5):\n model = ThreeBranchModelLSTM2(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_lstm_dir2_old / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_lstm2_old.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:53.709775Z","iopub.execute_input":"2025-09-02T05:22:53.70999Z","iopub.status.idle":"2025-09-02T05:22:56.479486Z","shell.execute_reply.started":"2025-09-02T05:22:53.709974Z","shell.execute_reply":"2025-09-02T05:22:56.478723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.2. All features - attention","metadata":{}},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass MultiHeadSelfAttention(nn.Module):\n def __init__(self, d_model, num_heads=8, dropout=0.1):\n super().__init__()\n assert d_model % num_heads == 0\n \n self.d_model = d_model\n self.num_heads = num_heads\n self.d_k = d_model // num_heads\n \n self.W_q = nn.Linear(d_model, d_model, bias=False)\n self.W_k = nn.Linear(d_model, d_model, bias=False)\n self.W_v = nn.Linear(d_model, d_model, bias=False)\n self.W_o = nn.Linear(d_model, d_model)\n \n self.dropout = nn.Dropout(dropout)\n self.layer_norm = nn.LayerNorm(d_model)\n \n def scaled_dot_product_attention(self, Q, K, V, mask=None):\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)\n \n if mask is not None:\n scores = scores.masked_fill(mask == 0, -1e9)\n \n attention_weights = F.softmax(scores, dim=-1)\n attention_weights = self.dropout(attention_weights)\n \n output = torch.matmul(attention_weights, V)\n return output, attention_weights\n \n def forward(self, x, mask=None):\n batch_size, seq_len, d_model = x.size()\n \n # Residual connection\n residual = x\n \n # Linear transformations and reshape\n Q = self.W_q(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n K = self.W_k(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n V = self.W_v(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n \n # Apply attention\n attn_output, _ = self.scaled_dot_product_attention(Q, K, V, mask)\n \n # Concatenate heads\n attn_output = attn_output.transpose(1, 2).contiguous().view(\n batch_size, seq_len, d_model)\n \n # Final linear layer\n output = self.W_o(attn_output)\n \n # Add residual connection and layer norm\n output = self.layer_norm(output + residual)\n \n return output\n\nclass PositionalEncoding(nn.Module):\n def __init__(self, d_model, max_length=5000):\n super().__init__()\n \n pe = torch.zeros(max_length, d_model)\n position = torch.arange(0, max_length, dtype=torch.float).unsqueeze(1)\n \n div_term = torch.exp(torch.arange(0, d_model, 2).float() * \n (-math.log(10000.0) / d_model))\n \n pe[:, 0::2] = torch.sin(position * div_term)\n pe[:, 1::2] = torch.cos(position * div_term)\n \n self.register_buffer('pe', pe.unsqueeze(0))\n \n def forward(self, x):\n return x + self.pe[:, :x.size(1)]\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n\nclass ImuModelAtt(nn.Module):\n def __init__(self, imu_dim):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branches\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n \n # Positional encoding for attention layers\n self.pos_encoding1 = PositionalEncoding(128)\n self.pos_encoding2 = PositionalEncoding(128)\n self.pos_encoding3 = PositionalEncoding(128)\n \n # Multi-head self-attention layers (replacing BiLSTM)\n self.self_attention1 = MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n self.self_attention2 = MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n self.self_attention3 = MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n \n # Additional attention layers for better feature learning\n self.attention_block1 = nn.Sequential(\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n )\n self.attention_block2 = nn.Sequential(\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n )\n self.attention_block3 = nn.Sequential(\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n )\n \n # Final attention for aggregation\n self.attention1 = AttentionLayer(128)\n self.attention2 = AttentionLayer(128)\n self.attention3 = AttentionLayer(128)\n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n # Branch 1: Process first 3 IMU dimensions\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n imu1 = self.pos_encoding1(imu1)\n attn_out1 = self.self_attention1(imu1)\n attn_out1 = self.attention_block1[0](attn_out1)\n attn_out1 = self.attention_block1[1](attn_out1)\n attended1 = self.attention1(attn_out1)\n\n # Branch 2: Process dimensions 3-15\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n imu2 = self.pos_encoding2(imu2)\n attn_out2 = self.self_attention2(imu2)\n attn_out2 = self.attention_block2[0](attn_out2)\n attn_out2 = self.attention_block2[1](attn_out2)\n attended2 = self.attention2(attn_out2)\n\n # Branch 3: Process remaining dimensions\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n imu3 = self.pos_encoding3(imu3)\n attn_out3 = self.self_attention3(imu3)\n attn_out3 = self.attention_block3[0](attn_out3)\n attn_out3 = self.attention_block3[1](attn_out3)\n attended3 = self.attention3(attn_out3)\n\n # Merge all branches\n merged = torch.cat((attended1, attended2, attended3), dim=1)\n\n return merged\n \nclass ThreeBranchModelAtt(nn.Module):\n def __init__(self, imu_dim, thm_dim, tof_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n self.thm_dim = thm_dim\n self.tof_dim = tof_dim\n \n # IMU branch\n self.imu_branch = ImuModelAtt(self.imu_dim)\n\n self.thm_branch = nn.Sequential(\n ResidualSEBlock(self.thm_dim, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n self.tof_branch = nn.Sequential(\n ResidualSEBlock(self.tof_dim, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n \n # Positional encoding for attention layers\n self.pos_encoding1 = PositionalEncoding(128)\n self.pos_encoding2 = PositionalEncoding(128)\n \n # Multi-head self-attention layers (replacing BiLSTM)\n self.self_attention1 = MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n self.self_attention2 = MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n \n # Additional attention layers for better feature learning\n self.attention_block1 = nn.Sequential(\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n )\n self.attention_block2 = nn.Sequential(\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(128, num_heads=8, dropout=0.4)\n )\n \n # Final attention for aggregation\n self.attention1 = AttentionLayer(128)\n self.attention2 = AttentionLayer(128)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(128*5, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n # Split input\n imu = x[:, :, :self.imu_dim]\n thm = x[:, :, self.imu_dim:(self.imu_dim+self.thm_dim)]\n tof = x[:, :, (self.imu_dim+self.thm_dim):]\n # Handle missing values (-1) by replacing with 500\n tof = torch.where(tof == -1, torch.full_like(tof, 500), tof)\n # Normalize to [0, 1] range\n tof = tof / 500.0\n \n # Process branches\n imu = self.imu_branch(imu)\n\n # Branch THM\n thm = self.thm_branch(thm.transpose(1, 2)).transpose(1, 2)\n thm = self.pos_encoding1(thm)\n attn_out1 = self.self_attention1(thm)\n attn_out1 = self.attention_block1[0](attn_out1)\n attn_out1 = self.attention_block1[1](attn_out1)\n attended1 = self.attention1(attn_out1)\n\n # Branch TOF\n tof = self.tof_branch(tof.transpose(1, 2)).transpose(1, 2)\n tof = self.pos_encoding2(tof)\n attn_out2 = self.self_attention2(tof)\n attn_out2 = self.attention_block2[0](attn_out2)\n attn_out2 = self.attention_block2[1](attn_out2)\n attended2 = self.attention2(attn_out2)\n\n # Merge branches\n merged = torch.cat((imu, attended1, attended2), dim=1)\n \n # Dense layers\n out = self.fc_layers(merged)\n return out # F.softmax(out, dim=1)\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:56.480409Z","iopub.execute_input":"2025-09-02T05:22:56.480631Z","iopub.status.idle":"2025-09-02T05:22:56.512394Z","shell.execute_reply.started":"2025-09-02T05:22:56.480615Z","shell.execute_reply":"2025-09-02T05:22:56.511624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_attention_dir / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_attention_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_attention = []\nfor fold in range(5):\n model = ThreeBranchModelAtt(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_attention_dir / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_attention.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:56.513321Z","iopub.execute_input":"2025-09-02T05:22:56.513604Z","iopub.status.idle":"2025-09-02T05:22:58.281183Z","shell.execute_reply.started":"2025-09-02T05:22:56.513574Z","shell.execute_reply":"2025-09-02T05:22:58.280369Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_attention_dir_old / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_attention_dir_old / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_attention_old = []\nfor fold in range(5):\n model = ThreeBranchModelAtt(len(imu_cols), len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_attention_dir_old / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_attention_old.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:22:58.282056Z","iopub.execute_input":"2025-09-02T05:22:58.282375Z","iopub.status.idle":"2025-09-02T05:23:00.0075Z","shell.execute_reply.started":"2025-09-02T05:22:58.28235Z","shell.execute_reply":"2025-09-02T05:23:00.006657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3.3. All features - CNN","metadata":{}},{"cell_type":"code","source":"class CoordAttention(nn.Module):\n \"\"\"\n Coordinate Attention for Sequences.\n Input Dimension: (B, T, C)\n Output Dimension: (B, T, C)\n \"\"\"\n def __init__(self, channels, reduction=4):\n super(CoordAttention, self).__init__()\n self.mid_channels = max(8, channels // reduction)\n\n self.compression = nn.Sequential(\n nn.Conv1d(channels, self.mid_channels, kernel_size=1, bias=False),\n nn.BatchNorm1d(self.mid_channels),\n nn.SiLU(inplace=True)\n )\n\n # Attention branches\n self.time_conv = nn.Conv1d(1, 1, kernel_size=5, padding=2, bias=False) \n self.channel_conv = nn.Conv1d(self.mid_channels, channels, kernel_size=1, bias=False)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n # x: (B, T, C)\n x_p = x.permute(0, 2, 1) # (B, C, T)\n f = self.compression(x_p) # (B, rC, T)\n\n ## Time Attention (B, 1, T)\n f_t = f.mean(dim=1, keepdim=True) \n time_attn = self.sigmoid(self.time_conv(f_t)) \n\n ## Channel Attention (B, C, 1)\n f_c = f.mean(dim=2, keepdim=True) \n channel_attn = self.sigmoid(self.channel_conv(f_c)) \n\n ## (B, T, C)\n out = (x_p * time_attn * channel_attn).permute(0,2,1)\n return out\n \nclass ResCNNBlock(nn.Module):\n \"\"\"\n Residual CNN Block with Coordinate Attention. \n Input Dimension: (B, T_in, C_in)\n Output Dimension: (B, T_out, C_out)\n \"\"\"\n def __init__(self, in_channels, out_channels, kernel_size=3, pool_size=2, drop=0.2):\n super(ResCNNBlock, self).__init__()\n\n ## CNN model\n self.cnn = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True),\n nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True)\n )\n ## Attention module\n self.attention = CoordAttention(out_channels)\n ## Projection path for skip connection\n self.shortcut_proj = None\n if in_channels != out_channels:\n self.shortcut_proj = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=1, padding='same', bias=False),\n nn.BatchNorm1d(out_channels)\n )\n ## Final processing components\n self.relu_final = nn.ReLU(inplace=True)\n if pool_size is not None: \n self.max_pool = nn.MaxPool1d(pool_size)\n else: \n self.max_pool = None\n self.dropout = nn.Dropout(drop)\n\n def forward(self, x):\n # (B, C_in, T)\n shortcut = x.permute(0,2,1)\n x_p = x.permute(0,2,1)\n \n out = self.cnn(x_p) # (B, C_out, T)\n out = self.attention(out.permute(0, 2, 1)) # (B, T, C_out)\n out = out.permute(0, 2, 1) # (B, C_out, T)\n \n ## Handle shortcut connection -> (B, C_out, T)\n if self.shortcut_proj:\n shortcut = self.shortcut_proj(shortcut)\n\n ## Residual connection\n out += shortcut\n out = self.relu_final(out)\n if self.max_pool is not None:\n ## (B, C_out, T) -> (B, C_out, T//pool_size)\n out = self.max_pool(out)\n out = self.dropout(out)\n ## (B, T_out, C_out)\n out = out.permute(0, 2, 1)\n return out\n \nclass TempCNNBlock(nn.Module):\n \"\"\"\n Temporal CNN Block with Dilated Convolution, Attention, and Skip Connection.\n Input Dimension: (B, T, C_in)\n Output Dimension: (B, T, C_out)\n \"\"\"\n def __init__(self, in_channels, out_channels, kernel_size=3, dilation=1, drop=0.25):\n super(TempCNNBlock, self).__init__()\n ## Padding computation\n padding = (kernel_size - 1) * dilation // 2\n ## Temporal Convolution block\n self.conv = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size,\n dilation=dilation, padding=padding, bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True)\n )\n ## Attention module\n self.attention = CoordAttention(out_channels)\n ## Dropout\n self.dropout = nn.Dropout(drop)\n \n ## Shortcut for skip connection\n self.shortcut_proj = None\n if in_channels != out_channels:\n self.shortcut_proj = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=1, padding='same', bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.relu_final = nn.ReLU()\n \n def forward(self, x):\n ## Processes skip connection\n # (B, C_in, T)\n x_p = x.permute(0,2,1)\n shortcut = x.permute(0,2,1)\n # (B, C_out, T)\n if self.shortcut_proj:\n shortcut = self.shortcut_proj(shortcut)\n ## Processes output\n out = self.conv(x_p) # (B, C_out, T)\n out = self.attention(out.permute(0,2,1)) # (B, T, C_out)\n out = out.permute(0,2,1) # (B, C_out, T)\n ## Skip connection\n out = self.relu_final(out + shortcut) # (B, C_out, T)\n out = self.dropout(out)\n return out.permute(0,2,1) # (B, T, C_out)\n \nclass MLPAttention(nn.Module):\n def __init__(self, feature_dim):\n super(MLPAttention, self).__init__()\n self.attn = nn.Sequential(\n nn.Linear(feature_dim, feature_dim//8),\n nn.SiLU(inplace=True),\n nn.Linear(feature_dim//8, 1)\n )\n\n def forward(self, x):\n # inputs shape: (B, T, C)\n weights = self.attn(x) # (B, T, 1)\n weights = F.softmax(weights, dim=1) # (B, T, 1)\n context = (x * weights).sum(dim=1) # (B, C)\n return context\n\n\nclass CrossAttention(nn.Module):\n \"\"\"\n Cross-attention module for multi-branch feature fusion.\n Allows each branch to attend to features from other branches.\n \"\"\"\n def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n super(CrossAttention, self).__init__()\n self.feature_dim = feature_dim\n self.num_heads = num_heads\n self.head_dim = feature_dim // num_heads\n \n assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n \n # Linear projections for queries, keys, and values for each branch\n self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n \n # Output projection\n self.out_linear = nn.Linear(feature_dim, feature_dim)\n self.dropout = nn.Dropout(dropout)\n \n # Layer normalization for residual connection\n self.layer_norm = nn.LayerNorm(feature_dim)\n \n def forward(self, query_branch, key_value_branches):\n \"\"\"\n Args:\n query_branch: (B, T, C) - the branch that will be updated\n key_value_branches: List of (B, T, C) tensors - branches to attend to\n Returns:\n Updated query_branch with cross-attention: (B, T, C)\n \"\"\"\n B, T, C = query_branch.shape\n \n # Create queries from the query branch\n Q = self.q_linear(query_branch) # (B, T, C)\n \n # Concatenate all key-value branches for attention\n all_kv = torch.stack(key_value_branches, dim=1) # (B, num_branches, T, C)\n num_branches = all_kv.shape[1]\n \n # Reshape for multi-head attention\n all_kv = all_kv.reshape(B, num_branches * T, C) # (B, num_branches*T, C)\n \n K = self.k_linear(all_kv) # (B, num_branches*T, C)\n V = self.v_linear(all_kv) # (B, num_branches*T, C)\n \n # Reshape for multi-head attention\n Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, T, head_dim)\n K = K.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n V = V.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n \n # Scaled dot-product attention\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # (B, num_heads, T, num_branches*T)\n attn_weights = F.softmax(scores, dim=-1)\n attn_weights = self.dropout(attn_weights)\n \n # Apply attention to values\n attn_output = torch.matmul(attn_weights, V) # (B, num_heads, T, head_dim)\n \n # Concatenate heads and put through final linear layer\n attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)\n output = self.out_linear(attn_output)\n \n # Residual connection and layer normalization\n output = self.layer_norm(query_branch + output)\n \n return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n \"\"\"\n Cross-attention fusion module for three IMU branches.\n Each branch attends to the other two branches.\n \"\"\"\n def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n super(IMUCrossAttentionFusion, self).__init__()\n \n # Cross-attention modules for each branch\n self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn4 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn5 = CrossAttention(feature_dim, num_heads, dropout)\n \n def forward(self, imu1, imu2, imu3, thm, tof):\n \"\"\"\n Args:\n imu1, imu2, imu3, thm, tof: (B, T, C) tensors from each branch\n Returns:\n Tuple of enhanced features: (enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof)\n \"\"\"\n # Each branch attends to the other two branches\n enhanced_imu1 = self.cross_attn1(imu1, [imu2, imu3, thm, tof])\n enhanced_imu2 = self.cross_attn2(imu2, [imu1, imu3, thm, tof])\n enhanced_imu3 = self.cross_attn3(imu3, [imu1, imu2, thm, tof])\n enhanced_thm = self.cross_attn4(thm, [imu1, imu2, imu3, tof])\n enhanced_tof = self.cross_attn5(tof, [imu1, imu2, imu3, thm])\n \n return enhanced_imu1, enhanced_imu2, enhanced_imu3, enhanced_thm, enhanced_tof\n \nclass ImuModelCNN(nn.Module):\n def __init__(self, imu_dim):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branches\n self.imu_branch1 = nn.Sequential(\n ResCNNBlock(self.imu_dim[0], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n self.imu_branch2 = nn.Sequential(\n ResCNNBlock(self.imu_dim[1], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n self.imu_branch3 = nn.Sequential(\n ResCNNBlock(self.imu_dim[2], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n\n def forward(self, x):\n imu = x[:, :, :sum(self.imu_dim)]\n\n # Branch 1\n imu1 = self.imu_branch1(imu[:, :, :self.imu_dim[0]])\n\n # Branch 2\n imu2 = self.imu_branch2(imu[:, :, self.imu_dim[0]:(self.imu_dim[0]+self.imu_dim[1])])\n\n # Branch 3\n imu3 = self.imu_branch3(imu[:, :, (self.imu_dim[0]+self.imu_dim[1]):])\n\n return imu1, imu2, imu3\n \nclass ThreeBranchModelCNN(nn.Module):\n def __init__(self, imu_dim, thm_dim, tof_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n self.thm_dim = thm_dim\n self.tof_dim = tof_dim\n self.n_classes = n_classes\n \n # IMU branch\n self.imu_branch = ImuModelCNN(self.imu_dim)\n\n self.thm_branch = nn.Sequential(\n ResCNNBlock(self.thm_dim, 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n\n self.tof_branch = nn.Sequential(\n ResCNNBlock(self.tof_dim, 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n\n #self.cross_attention_fusion = IMUCrossAttentionFusion(\n # feature_dim=256, num_heads=8, dropout=0.1\n #)\n \n self.merged_attention = CoordAttention(256*5)\n self.merged_dropout = nn.Dropout(1/3)\n self.temporalCNN = nn.Sequential(\n TempCNNBlock(256*5, 512, kernel_size=3, dilation=1, drop=0.2),\n TempCNNBlock(512, 512, kernel_size=3, dilation=2, drop=0.2),\n TempCNNBlock(512, 256, kernel_size=3, dilation=4, drop=0.2),\n TempCNNBlock(256, 256, kernel_size=3, dilation=8, drop=0.2),\n )\n\n self.classifier_attention = MLPAttention(256)\n self.classifier_dropout = nn.Dropout(0.2)\n self.classifier = nn.Sequential(\n nn.Linear(256, 128, bias=False),\n nn.BatchNorm1d(128),\n nn.ReLU(),\n nn.Dropout(0.2),\n nn.Linear(128, self.n_classes)\n )\n\n def forward(self, x):\n # Split input\n imu = x[:, :, :sum(self.imu_dim)]\n thm = x[:, :, sum(self.imu_dim):(sum(self.imu_dim)+self.thm_dim)]\n tof = x[:, :, (sum(self.imu_dim)+self.thm_dim):]\n \n # Handle missing values (-1) by replacing with 500\n tof = torch.where(tof == -1, torch.full_like(tof, 500), tof)\n # Normalize to [0, 1] range\n tof = tof / 500.0\n \n # Process branches\n imu1, imu2, imu3 = self.imu_branch(imu)\n\n # Branch THM\n thm = self.thm_branch(thm)\n\n # Branch TOF\n tof = self.tof_branch(tof)\n\n #imu1, imu2, imu3, thm, tof = self.cross_attention_fusion(imu1, imu2, imu3, thm, tof)\n\n # Merge branches\n merged = torch.cat((imu1, imu2, imu3, thm, tof), dim=-1)\n\n merged = self.merged_attention(merged)\n merged = self.merged_dropout(merged)\n out = self.temporalCNN(merged) \n\n ## Provides output to classifier\n out = self.classifier_attention(out) \n out = self.classifier_dropout(out)\n out = self.classifier(out) \n return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:00.008466Z","iopub.execute_input":"2025-09-02T05:23:00.008746Z","iopub.status.idle":"2025-09-02T05:23:00.04483Z","shell.execute_reply.started":"2025-09-02T05:23:00.008722Z","shell.execute_reply":"2025-09-02T05:23:00.044119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_cnn_dir / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_cnn_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_cnn = []\nfor fold in range(5):\n model = ThreeBranchModelCNN([12, 11, 12], len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_cnn_dir / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_cnn.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:00.045633Z","iopub.execute_input":"2025-09-02T05:23:00.046165Z","iopub.status.idle":"2025-09-02T05:23:03.371728Z","shell.execute_reply.started":"2025-09-02T05:23:00.046134Z","shell.execute_reply":"2025-09-02T05:23:03.370838Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = np.load(all_features_cnn_dir_old / \"feature_cols.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_cnn_dir_old / \"gesture_classes.npy\", allow_pickle=True)\n\nimu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\nthm_cols = [c for c in feature_cols if c.startswith('thm_')]\ntof_cols = [c for c in feature_cols if c.startswith('tof_')]\n\nmodels_all_features_cnn_old = []\nfor fold in range(5):\n model = ThreeBranchModelCNN([12, 11, 12], len(thm_cols), \n len(tof_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(all_features_cnn_dir_old / f\"{model_all_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_all_features_cnn_old.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:03.372586Z","iopub.execute_input":"2025-09-02T05:23:03.372875Z","iopub.status.idle":"2025-09-02T05:23:06.638815Z","shell.execute_reply.started":"2025-09-02T05:23:03.372856Z","shell.execute_reply":"2025-09-02T05:23:06.638004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models_all_features = models_all_features_lstm + models_all_features_lstm2 + \\\n models_all_features_attention + models_all_features_cnn\nlen(models_all_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:06.639708Z","iopub.execute_input":"2025-09-02T05:23:06.640151Z","iopub.status.idle":"2025-09-02T05:23:06.645795Z","shell.execute_reply.started":"2025-09-02T05:23:06.640124Z","shell.execute_reply":"2025-09-02T05:23:06.645058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models_all_features_old = models_all_features_lstm_old + models_all_features_lstm2_old + \\\n models_all_features_attention_old + models_all_features_cnn_old\nlen(models_all_features_old)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:06.646614Z","iopub.execute_input":"2025-09-02T05:23:06.646847Z","iopub.status.idle":"2025-09-02T05:23:06.660099Z","shell.execute_reply.started":"2025-09-02T05:23:06.646831Z","shell.execute_reply":"2025-09-02T05:23:06.659578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. IMU features models","metadata":{}},{"cell_type":"markdown","source":"## 4.1. IMU features - LSTM","metadata":{}},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n \nclass OneBranchModelLSTM(nn.Module):\n def __init__(self, imu_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branchs\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 64, 3, drop=0.3),\n ResidualSEBlock(64, 128, 5, drop=0.3)\n )\n \n # BiLSTM\n self.bilstm1 = nn.LSTM(128, 128, bidirectional=True, batch_first=True)\n self.dropout1 = nn.Dropout(0.4)\n self.bilstm2 = nn.LSTM(128, 128, bidirectional=True, batch_first=True)\n self.dropout2 = nn.Dropout(0.4)\n self.bilstm3 = nn.LSTM(128, 128, bidirectional=True, batch_first=True)\n self.dropout3 = nn.Dropout(0.4) \n \n # Attention\n self.attention1 = AttentionLayer(256)\n self.attention2 = AttentionLayer(256)\n self.attention3 = AttentionLayer(256)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(256*3, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n lstm_out1, _ = self.bilstm1(imu1)\n lstm_out1 = self.dropout1(lstm_out1)\n attended1 = self.attention1(lstm_out1)\n\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n lstm_out2, _ = self.bilstm2(imu2)\n lstm_out2 = self.dropout2(lstm_out2)\n attended2 = self.attention2(lstm_out2)\n\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n lstm_out3, _ = self.bilstm3(imu3)\n lstm_out3 = self.dropout3(lstm_out3)\n attended3 = self.attention3(lstm_out3)\n\n merged = torch.cat((attended1, attended2, attended3), dim=1)\n\n out = self.fc_layers(merged)\n return out ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:06.660934Z","iopub.execute_input":"2025-09-02T05:23:06.661166Z","iopub.status.idle":"2025-09-02T05:23:06.679325Z","shell.execute_reply.started":"2025-09-02T05:23:06.66115Z","shell.execute_reply":"2025-09-02T05:23:06.678754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imu_cols = np.load(imu_features_lstm_dir / \"feature_cols_imu.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(imu_features_lstm_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nmodels_imu_features_lstm = []\nfor fold in range(5):\n model = OneBranchModelLSTM(len(imu_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(imu_features_lstm_dir / f\"{model_imu_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_imu_features_lstm.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:06.679955Z","iopub.execute_input":"2025-09-02T05:23:06.680185Z","iopub.status.idle":"2025-09-02T05:23:07.712706Z","shell.execute_reply.started":"2025-09-02T05:23:06.680165Z","shell.execute_reply":"2025-09-02T05:23:07.711931Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n \nclass CrossAttention(nn.Module):\n \"\"\"\n Cross-attention module for multi-branch feature fusion.\n Allows each branch to attend to features from other branches.\n \"\"\"\n def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n super(CrossAttention, self).__init__()\n self.feature_dim = feature_dim\n self.num_heads = num_heads\n self.head_dim = feature_dim // num_heads\n \n assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n \n # Linear projections for queries, keys, and values for each branch\n self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n \n # Output projection\n self.out_linear = nn.Linear(feature_dim, feature_dim)\n self.dropout = nn.Dropout(dropout)\n \n # Layer normalization for residual connection\n self.layer_norm = nn.LayerNorm(feature_dim)\n \n def forward(self, query_branch, key_value_branches):\n \"\"\"\n Args:\n query_branch: (B, T, C) - the branch that will be updated\n key_value_branches: List of (B, T, C) tensors - branches to attend to\n Returns:\n Updated query_branch with cross-attention: (B, T, C)\n \"\"\"\n B, T, C = query_branch.shape\n \n # Create queries from the query branch\n Q = self.q_linear(query_branch) # (B, T, C)\n \n # Concatenate all key-value branches for attention\n all_kv = torch.stack(key_value_branches, dim=1) # (B, num_branches, T, C)\n num_branches = all_kv.shape[1]\n \n # Reshape for multi-head attention\n all_kv = all_kv.reshape(B, num_branches * T, C) # (B, num_branches*T, C)\n \n K = self.k_linear(all_kv) # (B, num_branches*T, C)\n V = self.v_linear(all_kv) # (B, num_branches*T, C)\n \n # Reshape for multi-head attention\n Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, T, head_dim)\n K = K.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n V = V.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n \n # Scaled dot-product attention\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # (B, num_heads, T, num_branches*T)\n attn_weights = F.softmax(scores, dim=-1)\n attn_weights = self.dropout(attn_weights)\n \n # Apply attention to values\n attn_output = torch.matmul(attn_weights, V) # (B, num_heads, T, head_dim)\n \n # Concatenate heads and put through final linear layer\n attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)\n output = self.out_linear(attn_output)\n \n # Residual connection and layer normalization\n output = self.layer_norm(query_branch + output)\n \n return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n \"\"\"\n Cross-attention fusion module for three IMU branches.\n Each branch attends to the other two branches.\n \"\"\"\n def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n super(IMUCrossAttentionFusion, self).__init__()\n \n # Cross-attention modules for each branch\n self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n \n def forward(self, imu1, imu2, imu3):\n \"\"\"\n Args:\n imu1, imu2, imu3: (B, T, C) tensors from each IMU branch\n Returns:\n Tuple of enhanced features: (enhanced_imu1, enhanced_imu2, enhanced_imu3)\n \"\"\"\n # Each branch attends to the other two branches\n enhanced_imu1 = self.cross_attn1(imu1, [imu2, imu3])\n enhanced_imu2 = self.cross_attn2(imu2, [imu1, imu3])\n enhanced_imu3 = self.cross_attn3(imu3, [imu1, imu2])\n \n return enhanced_imu1, enhanced_imu2, enhanced_imu3\n \nclass OneBranchModelLSTM2(nn.Module):\n def __init__(self, imu_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branchs\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n self.cross_attention_fusion = IMUCrossAttentionFusion(\n feature_dim=256, num_heads=8, dropout=0.1\n )\n\n # BiLSTM\n self.bilstm = nn.LSTM(256*3, 512, num_layers=1, bidirectional=True, batch_first=True)\n self.dropout = nn.Dropout(0.4) \n\n # Attention\n self.attention = AttentionLayer(1024)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(1024, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n\n # Apply cross-attention fusion between branches\n imu1, imu2, imu3 = self.cross_attention_fusion(imu1, imu2, imu3)\n\n merged = torch.cat((imu1, imu2, imu3), dim=2)\n\n lstm_out, _ = self.bilstm(merged)\n lstm_out = self.dropout(lstm_out)\n\n attended = self.attention(lstm_out)\n \n out = self.fc_layers(attended)\n return out ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:07.713568Z","iopub.execute_input":"2025-09-02T05:23:07.71376Z","iopub.status.idle":"2025-09-02T05:23:07.736Z","shell.execute_reply.started":"2025-09-02T05:23:07.713746Z","shell.execute_reply":"2025-09-02T05:23:07.735408Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imu_cols = np.load(imu_features_lstm_dir2 / \"feature_cols_imu.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(imu_features_lstm_dir2 / \"gesture_classes.npy\", allow_pickle=True)\n\nmodels_imu_features_lstm2 = []\nfor fold in range(5):\n model = OneBranchModelLSTM2(len(imu_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(imu_features_lstm_dir2 / f\"{model_imu_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_imu_features_lstm2.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:07.741109Z","iopub.execute_input":"2025-09-02T05:23:07.741354Z","iopub.status.idle":"2025-09-02T05:23:09.656782Z","shell.execute_reply.started":"2025-09-02T05:23:07.741336Z","shell.execute_reply":"2025-09-02T05:23:09.656004Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.2. IMU features - attention","metadata":{}},{"cell_type":"code","source":"class SEBlock(nn.Module):\n def __init__(self, channel, reduction=8):\n super().__init__()\n self.avg_pool = nn.AdaptiveAvgPool1d(1)\n self.fc = nn.Sequential(\n nn.Linear(channel, channel // reduction, bias=True),\n nn.ReLU(inplace=True),\n nn.Linear(channel // reduction, channel, bias=True),\n nn.Sigmoid()\n )\n\n def forward(self, x):\n b, c, _ = x.size()\n y = self.avg_pool(x).view(b, c)\n y = self.fc(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualSEBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, drop=0.3):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias = False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n self.se = SEBlock(out_channels)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(drop)\n self.pool = nn.MaxPool1d(2)\n \n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, padding='same', bias = False),\n nn.BatchNorm1d(out_channels)\n )\n else:\n self.shortcut = nn.Identity()\n\n def forward(self, x):\n identity = self.shortcut(x)\n \n out = self.conv1(x)\n out = self.bn1(out)\n out = self.relu(out)\n \n out = self.conv2(out)\n out = self.bn2(out)\n out = self.se(out)\n \n out += identity\n out = self.relu(out)\n out = self.pool(out)\n out = self.dropout(out)\n return out\n\nclass MultiHeadSelfAttention(nn.Module):\n def __init__(self, d_model, num_heads=8, dropout=0.1):\n super().__init__()\n assert d_model % num_heads == 0\n \n self.d_model = d_model\n self.num_heads = num_heads\n self.d_k = d_model // num_heads\n \n self.W_q = nn.Linear(d_model, d_model, bias=False)\n self.W_k = nn.Linear(d_model, d_model, bias=False)\n self.W_v = nn.Linear(d_model, d_model, bias=False)\n self.W_o = nn.Linear(d_model, d_model)\n \n self.dropout = nn.Dropout(dropout)\n self.layer_norm = nn.LayerNorm(d_model)\n \n def scaled_dot_product_attention(self, Q, K, V, mask=None):\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)\n \n if mask is not None:\n scores = scores.masked_fill(mask == 0, -1e9)\n \n attention_weights = F.softmax(scores, dim=-1)\n attention_weights = self.dropout(attention_weights)\n \n output = torch.matmul(attention_weights, V)\n return output, attention_weights\n \n def forward(self, x, mask=None):\n batch_size, seq_len, d_model = x.size()\n \n # Residual connection\n residual = x\n \n # Linear transformations and reshape\n Q = self.W_q(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n K = self.W_k(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n V = self.W_v(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)\n \n # Apply attention\n attn_output, _ = self.scaled_dot_product_attention(Q, K, V, mask)\n \n # Concatenate heads\n attn_output = attn_output.transpose(1, 2).contiguous().view(\n batch_size, seq_len, d_model)\n \n # Final linear layer\n output = self.W_o(attn_output)\n \n # Add residual connection and layer norm\n output = self.layer_norm(output + residual)\n \n return output\n\nclass PositionalEncoding(nn.Module):\n def __init__(self, d_model, max_length=5000):\n super().__init__()\n \n pe = torch.zeros(max_length, d_model)\n position = torch.arange(0, max_length, dtype=torch.float).unsqueeze(1)\n \n div_term = torch.exp(torch.arange(0, d_model, 2).float() * \n (-math.log(10000.0) / d_model))\n \n pe[:, 0::2] = torch.sin(position * div_term)\n pe[:, 1::2] = torch.cos(position * div_term)\n \n self.register_buffer('pe', pe.unsqueeze(0))\n \n def forward(self, x):\n return x + self.pe[:, :x.size(1)]\n\nclass AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Sequential(\n nn.Linear(hidden_dim, 1),\n nn.Tanh()\n )\n \n def forward(self, x):\n weights = self.attention(x).squeeze(-1)\n weights = F.softmax(weights, dim=1).unsqueeze(-1)\n context = torch.sum(x * weights, dim=1)\n return context\n\nclass OneBranchModelAtt(nn.Module):\n def __init__(self, imu_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n \n # IMU branches\n self.imu_branch1 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch2 = nn.Sequential(\n ResidualSEBlock(11, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n self.imu_branch3 = nn.Sequential(\n ResidualSEBlock(12, 128, 3, drop=0.3),\n ResidualSEBlock(128, 256, 5, drop=0.3)\n )\n \n # Positional encoding for attention layers\n self.pos_encoding1 = PositionalEncoding(256)\n self.pos_encoding2 = PositionalEncoding(256)\n self.pos_encoding3 = PositionalEncoding(256)\n \n # Multi-head self-attention layers (replacing BiLSTM)\n self.attention_block1 = nn.Sequential(\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4)\n )\n self.attention_block2 = nn.Sequential(\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4)\n )\n self.attention_block3 = nn.Sequential(\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4),\n MultiHeadSelfAttention(256, num_heads=8, dropout=0.4)\n )\n \n # Final attention for aggregation\n self.attention1 = AttentionLayer(256)\n self.attention2 = AttentionLayer(256)\n self.attention3 = AttentionLayer(256)\n \n # Dense layers\n self.fc_layers = nn.Sequential(\n nn.Linear(256*3, 512, bias = False),\n nn.BatchNorm1d(512),\n nn.ReLU(),\n nn.Dropout(0.5),\n nn.Linear(512, 256, bias = False),\n nn.BatchNorm1d(256),\n nn.ReLU(),\n nn.Dropout(0.3),\n nn.Linear(256, n_classes)\n )\n\n def forward(self, x):\n imu = x[:, :, :self.imu_dim]\n\n # Branch 1\n imu1 = self.imu_branch1(imu[:, :, :12].transpose(1, 2)).transpose(1, 2)\n imu1 = self.pos_encoding1(imu1)\n attn_out1 = self.attention_block1(imu1)\n attended1 = self.attention1(attn_out1)\n\n # Branch 2\n imu2 = self.imu_branch2(imu[:, :, 12:23].transpose(1, 2)).transpose(1, 2)\n imu2 = self.pos_encoding2(imu2)\n attn_out2 = self.attention_block2(imu2)\n attended2 = self.attention2(attn_out2)\n\n # Branch 3\n imu3 = self.imu_branch3(imu[:, :, 23:].transpose(1, 2)).transpose(1, 2)\n imu3 = self.pos_encoding3(imu3)\n attn_out3 = self.attention_block3(imu3)\n attended3 = self.attention3(attn_out3)\n\n # Merge all branches\n merged = torch.cat((attended1, attended2, attended3), dim=1)\n\n # Final classification\n out = self.fc_layers(merged)\n return out\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:09.657653Z","iopub.execute_input":"2025-09-02T05:23:09.6579Z","iopub.status.idle":"2025-09-02T05:23:09.68218Z","shell.execute_reply.started":"2025-09-02T05:23:09.657874Z","shell.execute_reply":"2025-09-02T05:23:09.681367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imu_cols = np.load(imu_features_attention_dir / \"feature_cols_imu.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(imu_features_attention_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nmodels_imu_features_attention = []\nfor fold in range(5):\n model = OneBranchModelAtt(len(imu_cols), len(gesture_classes)).to(device)\n state_dict = torch.load(imu_features_attention_dir / f\"{model_imu_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_imu_features_attention.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:09.683116Z","iopub.execute_input":"2025-09-02T05:23:09.683403Z","iopub.status.idle":"2025-09-02T05:23:11.658609Z","shell.execute_reply.started":"2025-09-02T05:23:09.683377Z","shell.execute_reply":"2025-09-02T05:23:11.657729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4.3. IMU features - CNN","metadata":{}},{"cell_type":"code","source":"class CoordAttention(nn.Module):\n \"\"\"\n Coordinate Attention for Sequences.\n Input Dimension: (B, T, C)\n Output Dimension: (B, T, C)\n \"\"\"\n def __init__(self, channels, reduction=4):\n super(CoordAttention, self).__init__()\n self.mid_channels = max(8, channels // reduction)\n\n self.compression = nn.Sequential(\n nn.Conv1d(channels, self.mid_channels, kernel_size=1, bias=False),\n nn.BatchNorm1d(self.mid_channels),\n nn.SiLU(inplace=True)\n )\n\n # Attention branches\n self.time_conv = nn.Conv1d(1, 1, kernel_size=5, padding=2, bias=False) \n self.channel_conv = nn.Conv1d(self.mid_channels, channels, kernel_size=1, bias=False)\n self.sigmoid = nn.Sigmoid()\n\n def forward(self, x):\n # x: (B, T, C)\n x_p = x.permute(0, 2, 1) # (B, C, T)\n f = self.compression(x_p) # (B, rC, T)\n\n ## Time Attention (B, 1, T)\n f_t = f.mean(dim=1, keepdim=True) \n time_attn = self.sigmoid(self.time_conv(f_t)) \n\n ## Channel Attention (B, C, 1)\n f_c = f.mean(dim=2, keepdim=True) \n channel_attn = self.sigmoid(self.channel_conv(f_c)) \n\n ## (B, T, C)\n out = (x_p * time_attn * channel_attn).permute(0,2,1)\n return out\n \nclass ResCNNBlock(nn.Module):\n \"\"\"\n Residual CNN Block with Coordinate Attention. \n Input Dimension: (B, T_in, C_in)\n Output Dimension: (B, T_out, C_out)\n \"\"\"\n def __init__(self, in_channels, out_channels, kernel_size=3, pool_size=2, drop=0.2):\n super(ResCNNBlock, self).__init__()\n\n ## CNN model\n self.cnn = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True),\n nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True)\n )\n ## Attention module\n self.attention = CoordAttention(out_channels)\n ## Projection path for skip connection\n self.shortcut_proj = None\n if in_channels != out_channels:\n self.shortcut_proj = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=1, padding='same', bias=False),\n nn.BatchNorm1d(out_channels)\n )\n ## Final processing components\n self.relu_final = nn.ReLU(inplace=True)\n if pool_size is not None: \n self.max_pool = nn.MaxPool1d(pool_size)\n else: \n self.max_pool = None\n self.dropout = nn.Dropout(drop)\n\n def forward(self, x):\n # (B, C_in, T)\n shortcut = x.permute(0,2,1)\n x_p = x.permute(0,2,1)\n \n out = self.cnn(x_p) # (B, C_out, T)\n out = self.attention(out.permute(0, 2, 1)) # (B, T, C_out)\n out = out.permute(0, 2, 1) # (B, C_out, T)\n \n ## Handle shortcut connection -> (B, C_out, T)\n if self.shortcut_proj:\n shortcut = self.shortcut_proj(shortcut)\n\n ## Residual connection\n out += shortcut\n out = self.relu_final(out)\n if self.max_pool is not None:\n ## (B, C_out, T) -> (B, C_out, T//pool_size)\n out = self.max_pool(out)\n out = self.dropout(out)\n ## (B, T_out, C_out)\n out = out.permute(0, 2, 1)\n return out\n \nclass TempCNNBlock(nn.Module):\n \"\"\"\n Temporal CNN Block with Dilated Convolution, Attention, and Skip Connection.\n Input Dimension: (B, T, C_in)\n Output Dimension: (B, T, C_out)\n \"\"\"\n def __init__(self, in_channels, out_channels, kernel_size=3, dilation=1, drop=0.25):\n super(TempCNNBlock, self).__init__()\n ## Padding computation\n padding = (kernel_size - 1) * dilation // 2\n ## Temporal Convolution block\n self.conv = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=kernel_size,\n dilation=dilation, padding=padding, bias=False),\n nn.BatchNorm1d(out_channels),\n nn.ReLU(inplace=True)\n )\n ## Attention module\n self.attention = CoordAttention(out_channels)\n ## Dropout\n self.dropout = nn.Dropout(drop)\n \n ## Shortcut for skip connection\n self.shortcut_proj = None\n if in_channels != out_channels:\n self.shortcut_proj = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, kernel_size=1, padding='same', bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.relu_final = nn.ReLU()\n \n def forward(self, x):\n ## Processes skip connection\n # (B, C_in, T)\n x_p = x.permute(0,2,1)\n shortcut = x.permute(0,2,1)\n # (B, C_out, T)\n if self.shortcut_proj:\n shortcut = self.shortcut_proj(shortcut)\n ## Processes output\n out = self.conv(x_p) # (B, C_out, T)\n out = self.attention(out.permute(0,2,1)) # (B, T, C_out)\n out = out.permute(0,2,1) # (B, C_out, T)\n ## Skip connection\n out = self.relu_final(out + shortcut) # (B, C_out, T)\n out = self.dropout(out)\n return out.permute(0,2,1) # (B, T, C_out)\n \nclass MLPAttention(nn.Module):\n def __init__(self, feature_dim):\n super(MLPAttention, self).__init__()\n self.attn = nn.Sequential(\n nn.Linear(feature_dim, feature_dim//8),\n nn.SiLU(inplace=True),\n nn.Linear(feature_dim//8, 1)\n )\n\n def forward(self, x):\n # inputs shape: (B, T, C)\n weights = self.attn(x) # (B, T, 1)\n weights = F.softmax(weights, dim=1) # (B, T, 1)\n context = (x * weights).sum(dim=1) # (B, C)\n return context\n\n\nclass CrossAttention(nn.Module):\n \"\"\"\n Cross-attention module for multi-branch feature fusion.\n Allows each branch to attend to features from other branches.\n \"\"\"\n def __init__(self, feature_dim, num_heads=8, dropout=0.1):\n super(CrossAttention, self).__init__()\n self.feature_dim = feature_dim\n self.num_heads = num_heads\n self.head_dim = feature_dim // num_heads\n \n assert feature_dim % num_heads == 0, \"feature_dim must be divisible by num_heads\"\n \n # Linear projections for queries, keys, and values for each branch\n self.q_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.k_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n self.v_linear = nn.Linear(feature_dim, feature_dim, bias=False)\n \n # Output projection\n self.out_linear = nn.Linear(feature_dim, feature_dim)\n self.dropout = nn.Dropout(dropout)\n \n # Layer normalization for residual connection\n self.layer_norm = nn.LayerNorm(feature_dim)\n \n def forward(self, query_branch, key_value_branches):\n \"\"\"\n Args:\n query_branch: (B, T, C) - the branch that will be updated\n key_value_branches: List of (B, T, C) tensors - branches to attend to\n Returns:\n Updated query_branch with cross-attention: (B, T, C)\n \"\"\"\n B, T, C = query_branch.shape\n \n # Create queries from the query branch\n Q = self.q_linear(query_branch) # (B, T, C)\n \n # Concatenate all key-value branches for attention\n all_kv = torch.stack(key_value_branches, dim=1) # (B, num_branches, T, C)\n num_branches = all_kv.shape[1]\n \n # Reshape for multi-head attention\n all_kv = all_kv.reshape(B, num_branches * T, C) # (B, num_branches*T, C)\n \n K = self.k_linear(all_kv) # (B, num_branches*T, C)\n V = self.v_linear(all_kv) # (B, num_branches*T, C)\n \n # Reshape for multi-head attention\n Q = Q.view(B, T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, T, head_dim)\n K = K.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n V = V.view(B, num_branches * T, self.num_heads, self.head_dim).transpose(1, 2) # (B, num_heads, num_branches*T, head_dim)\n \n # Scaled dot-product attention\n scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim) # (B, num_heads, T, num_branches*T)\n attn_weights = F.softmax(scores, dim=-1)\n attn_weights = self.dropout(attn_weights)\n \n # Apply attention to values\n attn_output = torch.matmul(attn_weights, V) # (B, num_heads, T, head_dim)\n \n # Concatenate heads and put through final linear layer\n attn_output = attn_output.transpose(1, 2).contiguous().view(B, T, C) # (B, T, C)\n output = self.out_linear(attn_output)\n \n # Residual connection and layer normalization\n output = self.layer_norm(query_branch + output)\n \n return output\n\nclass IMUCrossAttentionFusion(nn.Module):\n \"\"\"\n Cross-attention fusion module for three IMU branches.\n Each branch attends to the other two branches.\n \"\"\"\n def __init__(self, feature_dim=256, num_heads=8, dropout=0.1):\n super(IMUCrossAttentionFusion, self).__init__()\n \n # Cross-attention modules for each branch\n self.cross_attn1 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn2 = CrossAttention(feature_dim, num_heads, dropout)\n self.cross_attn3 = CrossAttention(feature_dim, num_heads, dropout)\n \n def forward(self, imu1, imu2, imu3):\n \"\"\"\n Args:\n imu1, imu2, imu3: (B, T, C) tensors from each IMU branch\n Returns:\n Tuple of enhanced features: (enhanced_imu1, enhanced_imu2, enhanced_imu3)\n \"\"\"\n # Each branch attends to the other two branches\n enhanced_imu1 = self.cross_attn1(imu1, [imu2, imu3])\n enhanced_imu2 = self.cross_attn2(imu2, [imu1, imu3])\n enhanced_imu3 = self.cross_attn3(imu3, [imu1, imu2])\n \n return enhanced_imu1, enhanced_imu2, enhanced_imu3\n\nclass OneBranchModelCNN(nn.Module):\n def __init__(self, imu_dim, n_classes):\n super().__init__()\n self.imu_dim = imu_dim\n self.n_classes = n_classes\n \n # IMU branches\n self.imu_branch1 = nn.Sequential(\n ResCNNBlock(imu_dim[0], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n self.imu_branch2 = nn.Sequential(\n ResCNNBlock(imu_dim[1], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n self.imu_branch3 = nn.Sequential(\n ResCNNBlock(imu_dim[2], 128, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(128, 256, kernel_size=3, pool_size=2, drop=0.2),\n ResCNNBlock(256, 256, kernel_size=5, pool_size=None, drop=0.2),\n )\n\n self.cross_attention_fusion = IMUCrossAttentionFusion(\n feature_dim=256, num_heads=8, dropout=0.1\n )\n\n self.merged_attention = CoordAttention(256*3)\n self.merged_dropout = nn.Dropout(1/3)\n self.temporalCNN = nn.Sequential(\n TempCNNBlock(256*3, 512, kernel_size=3, dilation=1, drop=0.2),\n TempCNNBlock(512, 512, kernel_size=3, dilation=2, drop=0.2),\n TempCNNBlock(512, 256, kernel_size=3, dilation=4, drop=0.2),\n TempCNNBlock(256, 256, kernel_size=3, dilation=8, drop=0.2),\n )\n\n self.classifier_attention = MLPAttention(256)\n self.classifier_dropout = nn.Dropout(0.2)\n self.classifier = nn.Sequential(\n nn.Linear(256, 128, bias=False),\n nn.BatchNorm1d(128),\n nn.ReLU(),\n nn.Dropout(0.2),\n nn.Linear(128, self.n_classes)\n )\n\n def forward(self, x):\n imu = x[:, :, :sum(self.imu_dim)]\n\n # Branch 1\n imu1 = self.imu_branch1(imu[:, :, :self.imu_dim[0]])\n\n # Branch 2\n imu2 = self.imu_branch2(imu[:, :, self.imu_dim[0]:(self.imu_dim[0]+self.imu_dim[1])])\n\n # Branch 3\n imu3 = self.imu_branch3(imu[:, :, (self.imu_dim[0]+self.imu_dim[1]):])\n\n # Apply cross-attention fusion between branches\n enhanced_imu1, enhanced_imu2, enhanced_imu3 = self.cross_attention_fusion(imu1, imu2, imu3)\n\n # Merge all branches\n x_merged = torch.cat((enhanced_imu1, enhanced_imu2, enhanced_imu3), dim=2)\n x_merged = self.merged_attention(x_merged)\n x_merged = self.merged_dropout(x_merged)\n out = self.temporalCNN(x_merged) # (B, 37, 128)\n\n ## Provides output to classifier\n out = self.classifier_attention(out) # (B, 128)\n out = self.classifier_dropout(out)\n out = self.classifier(out) # (B, 18)\n return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:11.659649Z","iopub.execute_input":"2025-09-02T05:23:11.659987Z","iopub.status.idle":"2025-09-02T05:23:11.690987Z","shell.execute_reply.started":"2025-09-02T05:23:11.659967Z","shell.execute_reply":"2025-09-02T05:23:11.690265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imu_cols = np.load(imu_features_cnn_dir / \"feature_cols_imu.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(imu_features_cnn_dir / \"gesture_classes.npy\", allow_pickle=True)\n\nmodels_imu_features_cnn = []\nfor fold in range(5):\n model = OneBranchModelCNN([12, 11, 12], len(gesture_classes)).to(device)\n state_dict = torch.load(imu_features_cnn_dir / f\"{model_imu_features_path}{fold}.pt\", \n map_location=device, weights_only=True)\n model.load_state_dict(state_dict)\n model = model.eval()\n models_imu_features_cnn.append(model)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:11.691851Z","iopub.execute_input":"2025-09-02T05:23:11.692075Z","iopub.status.idle":"2025-09-02T05:23:13.905797Z","shell.execute_reply.started":"2025-09-02T05:23:11.692052Z","shell.execute_reply":"2025-09-02T05:23:13.905038Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"models_imu_features = models_imu_features_lstm + models_imu_features_lstm2 + \\\n models_imu_features_attention + models_imu_features_cnn\nlen(models_imu_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:13.906698Z","iopub.execute_input":"2025-09-02T05:23:13.906936Z","iopub.status.idle":"2025-09-02T05:23:13.911977Z","shell.execute_reply.started":"2025-09-02T05:23:13.906918Z","shell.execute_reply":"2025-09-02T05:23:13.911354Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Devin's models","metadata":{}},{"cell_type":"code","source":"import polars as pl\nimport numpy as np\nimport glob\nimport os\n\nfrom pickle import load\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport kaggle_evaluation.cmi_inference_server\n\nfrom scipy.spatial.transform import Rotation as R","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:13.912867Z","iopub.execute_input":"2025-09-02T05:23:13.91332Z","iopub.status.idle":"2025-09-02T05:23:14.202889Z","shell.execute_reply.started":"2025-09-02T05:23:13.913294Z","shell.execute_reply":"2025-09-02T05:23:14.202122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device2 = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.203809Z","iopub.execute_input":"2025-09-02T05:23:14.204029Z","iopub.status.idle":"2025-09-02T05:23:14.208454Z","shell.execute_reply.started":"2025-09-02T05:23:14.20401Z","shell.execute_reply":"2025-09-02T05:23:14.207508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imu_cols = [\"acc_x\",\"acc_y\", \"acc_z\", \"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\ndiff_cols = [\"diff_x\", \"diff_y\", \"diff_z\"]\nextra_imu_cols = [\"mag\", \"angle\"]\nextra_extra_imu = [\"ang_vel_x\", \"ang_vel_y\", \"ang_vel_z\", \"ang_dist\"]\nthm_cols = [\"thm_\" + str(x) for x in range(1, 6)]\nthm_diff_cols = [\"thm_\" + str(x) +\"_diff\" for x in range(1, 6)] \ntof_cols = [\"tof_agg\" + str(x) for x in range(0,45)]\ntof_diff_cols = [\"tof_agg\" + str(x) + \"_diff\" for x in range(0,45)]\n\nall_model_cols = imu_cols+diff_cols+extra_imu_cols+extra_extra_imu+thm_cols+tof_cols\nall_model_diff_cols = imu_cols+diff_cols+extra_imu_cols+extra_extra_imu+thm_cols+tof_cols+thm_diff_cols+tof_diff_cols\nimu_model_cols = imu_cols+diff_cols+extra_imu_cols+extra_extra_imu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.209254Z","iopub.execute_input":"2025-09-02T05:23:14.209553Z","iopub.status.idle":"2025-09-02T05:23:14.221613Z","shell.execute_reply.started":"2025-09-02T05:23:14.209535Z","shell.execute_reply":"2025-09-02T05:23:14.220947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# only issue with this is its either all missing or none, \ndef handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [1.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n\n# thanks: https://www.kaggle.com/code/nksusth/lb-0-78-quaternions-tf-bilstm-gru-attention\ndef calculate_angular_velocity_from_quat2(rot_data, time_delta=1/10): # Assuming 10Hz sampling rate\n if isinstance(rot_data, pl.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].to_numpy()\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n\n for i in range(num_samples - 1):\n q_t = quat_values[i]\n q_t_plus_dt = quat_values[i+1]\n\n if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n continue\n\n try:\n rot_t = R.from_quat(q_t)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n\n # Calculate the relative rotation\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n # Convert delta rotation to angular velocity vector\n # The rotation vector (Euler axis * angle) scaled by 1/dt\n # is a good approximation for small delta_rot\n angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n except ValueError:\n # If quaternion is invalid, angular velocity remains zero\n pass\n \n return angular_vel\n\n# thanks: https://www.kaggle.com/code/nksusth/lb-0-78-quaternions-tf-bilstm-gru-attention\ndef calculate_angular_distance2(rot_data):\n if isinstance(rot_data, pl.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].to_numpy()\n else:\n quat_values = rot_data\n\n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n\n for i in range(num_samples - 1):\n q1 = quat_values[i]\n q2 = quat_values[i+1]\n\n if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n angular_dist[i] = 0 # Или np.nan, в зависимости от желаемого поведения\n continue\n try:\n # Преобразование кватернионов в объекты Rotation\n r1 = R.from_quat(q1)\n r2 = R.from_quat(q2)\n\n # Вычисление углового расстояния: 2 * arccos(|real(p * q*)|)\n # где p* - сопряженный кватернион q\n # В scipy.spatial.transform.Rotation, r1.inv() * r2 дает относительное вращение.\n # Угол этого относительного вращения - это и есть угловое расстояние.\n relative_rotation = r1.inv() * r2\n \n # Угол rotation vector соответствует угловому расстоянию\n # Норма rotation vector - это угол в радианах\n angle = np.linalg.norm(relative_rotation.as_rotvec())\n angular_dist[i] = angle\n except ValueError:\n angular_dist[i] = 0 # В случае недействительных кватернионов\n pass\n \n return angular_dist\n\ndef preprocess_left_handed(l_tr):\n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n print(\"yes\")\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.222467Z","iopub.execute_input":"2025-09-02T05:23:14.222716Z","iopub.status.idle":"2025-09-02T05:23:14.245093Z","shell.execute_reply.started":"2025-09-02T05:23:14.222691Z","shell.execute_reply":"2025-09-02T05:23:14.244324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prep_data(df, target_cols, scaler):\n\n df = df.with_columns(pl.col(\"acc_x\").diff().alias(\"diff_x\"))\n df = df.with_columns(pl.col(\"acc_y\").diff().alias(\"diff_y\"))\n df = df.with_columns(pl.col(\"acc_z\").diff().alias(\"diff_z\"))\n df = df.with_columns((pl.col(\"acc_x\").pow(2) + pl.col(\"acc_y\").pow(2) + pl.col(\"acc_z\").pow(2)).sqrt().alias(\"mag\"))\n df = df.with_columns((2*pl.col(\"rot_w\").arccos()).alias(\"angle\"))\n\n ang_vel = calculate_angular_velocity_from_quat2(df)\n ang_dist = calculate_angular_distance2(df)\n df = df.with_columns(pl.DataFrame(ang_vel, schema=[\"ang_vel_x\", \"ang_vel_y\", \"ang_vel_z\"]))\n df = df.with_columns(pl.Series(ang_dist).alias(\"ang_dist\"))\n\n\n \n if len(target_cols) > 20:\n groups1 = [[0,1,8,9],[3,4,11,12],[6,7,14,15],[24,25,32,33],[27,28,35,36],\n [30,31,38,39],[48,49,56,57], [51,52,59,60], [54,55,62,63]]\n \n cols1 = []\n for i in groups1:\n for k in range(1, 6):\n c = list()\n for j in i:\n c.append(\"tof_\"+str(k)+\"_v\"+str(j))\n cols1.append(c) \n \n for e, cs in enumerate(cols1):\n t = df.select(pl.col(cs))\n t = t.with_columns(pl.all().replace(-1, None))\n t = t.with_columns(pl.mean_horizontal(pl.all()).alias(\"tof_agg\" + str(e)))\n df = df.hstack(pl.DataFrame(t[\"tof_agg\" + str(e)]))\n\n for i in range(1,6):\n df = df.with_columns(pl.col(f\"thm_{i}\").diff().alias(f\"thm_{i}_diff\"))\n for i in range(45):\n df = df.with_columns(pl.col(f\"tof_agg{i}\").diff().alias(f\"tof_agg{i}_diff\"))\n \n df = df.with_columns(pl.col(target_cols).replace([float(\"inf\"), float(\"-inf\"), float(\"nan\")], None))\n \n scaled = pl.DataFrame(scaler.transform(df[target_cols]), schema=target_cols)\n df = df.with_columns(scaled)\n \n \n \n df = df.with_columns(pl.col(target_cols).replace([float(\"inf\"), float(\"-inf\"), float(\"nan\")], None))\n df = df.fill_null(0)\n \n seq_length = 128+5\n x_ts = np.zeros((1,len(target_cols), seq_length))\n \n x_ts[0,:,-df.shape[0]:] = df[target_cols].to_numpy()[-seq_length:,:].T\n return x_ts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.245942Z","iopub.execute_input":"2025-09-02T05:23:14.246201Z","iopub.status.idle":"2025-09-02T05:23:14.262996Z","shell.execute_reply.started":"2025-09-02T05:23:14.246184Z","shell.execute_reply":"2025-09-02T05:23:14.262424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"/kaggle/input/cmi-detect-behavior/encoder.pkl\", \"rb\") as f:\n encoder = load(f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.263853Z","iopub.execute_input":"2025-09-02T05:23:14.26464Z","iopub.status.idle":"2025-09-02T05:23:14.283705Z","shell.execute_reply.started":"2025-09-02T05:23:14.264621Z","shell.execute_reply":"2025-09-02T05:23:14.283074Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class ResidualBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn1 = nn.BatchNorm1d(num_features=out_channels)\n self.conv2 = nn.Conv1d(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n \n if in_channels != out_channels:\n self.res_layer = torch.nn.Conv1d(in_channels, out_channels, kernel_size=1, padding=\"same\")\n else:\n self.res_layer = None\n \n def forward(self, x):\n res = x\n x = F.relu(self.bn1(self.conv1(x)))\n x = F.relu(self.bn2(self.conv2(x)))\n if self.res_layer is None:\n x = torch.add(x, res)\n else: \n x = torch.add(x, self.res_layer(res))\n return x \n\nclass SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\nclass ResidualBlockSE(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size):\n super().__init__()\n self.conv1 = nn.Conv1d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn1 = nn.BatchNorm1d(num_features=out_channels)\n self.conv2 = nn.Conv1d(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn2 = nn.BatchNorm1d(num_features=out_channels)\n self.se = SEBlock(out_channels)\n \n if in_channels != out_channels:\n self.res_layer = nn.Conv1d(in_channels, out_channels, kernel_size=1, padding=\"same\")\n self.res_bn = nn.BatchNorm1d(out_channels)\n else:\n self.res_layer = None\n self.res_bn = None\n \n def forward(self, x):\n res = x\n x = F.relu(self.bn1(self.conv1(x)))\n x = self.bn2(self.conv2(x))\n x = self.se(x)\n if self.res_layer is None:\n x = torch.add(x, res)\n else: \n x = torch.add(x, self.res_bn(self.res_layer(res)))\n x = F.relu(x)\n return x \n\n\nclass RNNetAllBig(nn.Module):\n def __init__(self, chan=128,n_all_feat=50, classes=18):\n super().__init__()\n\n self.block0_0 = ResidualBlock(4, chan//2, 3)\n self.block0_1 = ResidualBlock(chan//2, chan//2, 3)\n self.block0_2 = ResidualBlock(chan//2, chan//2, 3)\n\n self.block1_0 = ResidualBlock(5, chan//2, 3)\n self.block1_1 = ResidualBlock(chan//2, chan//2, 3)\n self.block1_2 = ResidualBlock(chan//2, chan//2, 3)\n\n self.block2_0 = ResidualBlock(3, chan//2, 3)\n self.block2_1 = ResidualBlock(chan//2, chan//2, 3)\n self.block2_2 = ResidualBlock(chan//2, chan//2, 3)\n\n self.block3_0 = ResidualBlock(4, chan//2,3)\n self.block3_1 = ResidualBlock(chan//2, chan//2,3)\n self.block3_2 = ResidualBlock(chan//2, chan//2, 3)\n\n self.block4_0 = ResidualBlock(n_all_feat, 2*chan,3)\n self.block4_1 = ResidualBlock(2*chan, 2*chan,5)\n\n self.block5_0 = ResidualBlock(16, chan,3)\n self.block5_1 = ResidualBlock(chan, chan,3)\n self.block5_2 = ResidualBlock(chan, chan, 3)\n \n self.rnn = nn.GRU(5*chan, 512,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n self.fc0 = nn.Linear(1024, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, classes)\n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:16,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n x4 = self.block4_1(self.block4_0(x[:,16:,:]))\n x5 = self.block5_2(self.block5_1(self.block5_0(x[:,:16,:])))\n \n x = torch.cat([x0,x1,x2, x3, x4, x5], dim=1)\n \n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x \n\nclass RNNetAllBigSE(nn.Module):\n def __init__(self, chan=128, n_all_feat=50):\n super().__init__()\n\n self.block0_0 = ResidualBlockSE(4, chan//2, 3)\n self.block0_1 = ResidualBlockSE(chan//2, chan//2, 3)\n self.block0_2 = ResidualBlockSE(chan//2, chan//2, 3)\n\n self.block1_0 = ResidualBlockSE(5, chan//2, 3)\n self.block1_1 = ResidualBlockSE(chan//2, chan//2, 3)\n self.block1_2 = ResidualBlockSE(chan//2, chan//2, 3)\n\n self.block2_0 = ResidualBlockSE(3, chan//2, 3)\n self.block2_1 = ResidualBlockSE(chan//2, chan//2, 3)\n self.block2_2 = ResidualBlockSE(chan//2, chan//2, 3)\n\n self.block3_0 = ResidualBlockSE(4, chan//2,3)\n self.block3_1 = ResidualBlockSE(chan//2, chan//2,3)\n self.block3_2 = ResidualBlockSE(chan//2, chan//2, 3)\n\n self.block4_0 = ResidualBlockSE(n_all_feat, 2*chan,3)\n self.block4_1 = ResidualBlockSE(2*chan, 2*chan,5)\n\n self.block5_0 = ResidualBlockSE(16, chan,3)\n self.block5_1 = ResidualBlockSE(chan, chan,3)\n self.block5_2 = ResidualBlockSE(chan, chan, 3)\n \n self.rnn = nn.GRU(5*chan, 512,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n self.fc0 = nn.Linear(1024, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, 18)\n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:16,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n x4 = self.block4_1(self.block4_0(x[:,16:,:]))\n x5 = self.block5_2(self.block5_1(self.block5_0(x[:,:16,:])))\n \n x = torch.cat([x0,x1,x2, x3, x4, x5], dim=1)\n \n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x \n\n\n# https://www.kaggle.com/code/takoihiraokazu/cv-ensemble-sub-0607-1\nclass RnnModelAlldata(nn.Module):\n def __init__(\n self, dropout=0.2,\n input_numerical_size=66,\n numeraical_linear_size = 128,\n model_size = 128,\n linear_out = 128,\n out_size=18):\n super(RnnModelAlldata, self).__init__()\n self.numerical_linear = nn.Sequential(\n nn.Linear(input_numerical_size, numeraical_linear_size),\n nn.LayerNorm(numeraical_linear_size)\n )\n \n self.rnn = nn.GRU(numeraical_linear_size, model_size,\n num_layers = 2, \n bidirectional=True,\n batch_first=True)\n self.linear_out = nn.Sequential(\n nn.Linear(model_size*2, \n linear_out),\n nn.LayerNorm(linear_out),\n nn.Dropout(dropout),\n nn.Linear(linear_out, \n out_size))\n self._reinitialize()\n \n def _reinitialize(self):\n \"\"\"\n Tensorflow/Keras-like initialization\n \"\"\"\n for name, p in self.named_parameters():\n if 'rnn' in name:\n if 'weight_ih' in name:\n nn.init.xavier_uniform_(p.data)\n elif 'weight_hh' in name:\n nn.init.orthogonal_(p.data)\n elif 'bias_ih' in name:\n p.data.fill_(0)\n # Set forget-gate bias to 1\n n = p.size(0)\n p.data[(n // 4):(n // 2)].fill_(1)\n elif 'bias_hh' in name:\n p.data.fill_(0)\n \n def forward(self, numerical_array):\n numerical_array = numerical_array.permute(0, 2, 1)\n numerical_embedding = self.numerical_linear(numerical_array)\n output,_ = self.rnn(numerical_embedding)\n output = self.linear_out(output)\n return output[:,-30:,:].mean(axis=1)\n\nclass NetAll(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4,n_all_feat=50):\n super().__init__()\n \n self.block0_0 = ResidualBlock(4, chan,3)\n self.block0_1 = ResidualBlock(chan, chan,3)\n self.block0_2 = ResidualBlock(chan, chan, 3)\n\n self.block1_0 = ResidualBlock(5, chan,3)\n self.block1_1 = ResidualBlock(chan, chan,3)\n self.block1_2 = ResidualBlock(chan, chan, 3)\n\n self.block2_0 = ResidualBlock(3, chan,3)\n self.block2_1 = ResidualBlock(chan, chan,3)\n self.block2_2 = ResidualBlock(chan, chan, 3)\n\n self.block3_0 = ResidualBlock(4, chan,3)\n self.block3_1 = ResidualBlock(chan, chan,3)\n self.block3_2 = ResidualBlock(chan, chan, 3)\n\n self.block4_0 = ResidualBlock(n_all_feat, 2*chan,3)\n self.block4_1 = ResidualBlock(2*chan, 2*chan,5)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualBlock(6*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualBlock((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:16,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n x4 = self.block4_1(self.block4_0(x[:,16:,:]))\n \n x = torch.cat([x0,x1,x2, x3, x4], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x \n\nclass NetAllSE(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4,n_all_feat=50):\n super().__init__()\n \n self.block0_0 = ResidualBlockSE(4, chan,3)\n self.block0_1 = ResidualBlockSE(chan, chan,3)\n self.block0_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block1_0 = ResidualBlockSE(5, chan,3)\n self.block1_1 = ResidualBlockSE(chan, chan,3)\n self.block1_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block2_0 = ResidualBlockSE(3, chan,3)\n self.block2_1 = ResidualBlockSE(chan, chan,3)\n self.block2_2 = ResidualBlock(chan, chan, 3)\n\n self.block3_0 = ResidualBlockSE(4, chan,3)\n self.block3_1 = ResidualBlockSE(chan, chan,3)\n self.block3_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block4_0 = ResidualBlockSE(n_all_feat, 2*chan,3)\n self.block4_1 = ResidualBlockSE(2*chan, 2*chan,5)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualBlockSE(6*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualBlockSE((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:16,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n x4 = self.block4_1(self.block4_0(x[:,16:,:]))\n \n x = torch.cat([x0,x1,x2, x3, x4], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x \n\n\n\n\n\n\nclass RNNetTOFTHM(nn.Module):\n def __init__(self, chan=128):\n super().__init__()\n\n self.block4_0 = ResidualBlock(50, 2*chan,3)\n self.block4_1 = ResidualBlock(2*chan, 2*chan,5)\n \n self.rnn = nn.GRU(2*chan, 128,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n self.fc0 = nn.Linear(256, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, 18)\n def forward(self, x):\n x = self.block4_1(self.block4_0(x[:,16:,:]))\n \n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x \n\n\nclass RNNetImu(nn.Module):\n def __init__(self, chan=64):\n super().__init__()\n\n self.block0_0 = ResidualBlock(4, chan,3)\n self.block0_1 = ResidualBlock(chan, chan,3)\n self.block0_2 = ResidualBlock(chan, chan, 3)\n\n self.block1_0 = ResidualBlock(5, chan,3)\n self.block1_1 = ResidualBlock(chan, chan,3)\n self.block1_2 = ResidualBlock(chan, chan, 3)\n\n self.block2_0 = ResidualBlock(3, chan,3)\n self.block2_1 = ResidualBlock(chan, chan,3)\n self.block2_2 = ResidualBlock(chan, chan, 3)\n\n self.block3_0 = ResidualBlock(4, chan,3)\n self.block3_1 = ResidualBlock(chan, chan,3)\n self.block3_2 = ResidualBlock(chan, chan, 3)\n \n self.rnn = nn.GRU(4*chan, 128,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n\n self.fc0 = nn.Linear(256, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n \n x = torch.cat([x0,x1,x2,x3], dim=1)\n \n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x \n\n\n\nclass NetImu(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4):\n super().__init__()\n \n self.block0_0 = ResidualBlock(4, chan,3)\n self.block0_1 = ResidualBlock(chan, chan,3)\n self.block0_2 = ResidualBlock(chan, chan, 3)\n\n self.block1_0 = ResidualBlock(5, chan,3)\n self.block1_1 = ResidualBlock(chan, chan,3)\n self.block1_2 = ResidualBlock(chan, chan, 3)\n\n self.block2_0 = ResidualBlock(3, chan,3)\n self.block2_1 = ResidualBlock(chan, chan,3)\n self.block2_2 = ResidualBlock(chan, chan, 3)\n\n self.block3_0 = ResidualBlock(4, chan,3)\n self.block3_1 = ResidualBlock(chan, chan,3)\n self.block3_2 = ResidualBlock(chan, chan, 3)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualBlock(4*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualBlock((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n \n x = torch.cat([x0,x1,x2,x3], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x \n\nclass RNNetImuSE(nn.Module):\n def __init__(self, chan=64):\n super().__init__()\n\n self.block0_0 = ResidualBlockSE(4, chan,3)\n self.block0_1 = ResidualBlockSE(chan, chan,3)\n self.block0_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block1_0 = ResidualBlockSE(5, chan,3)\n self.block1_1 = ResidualBlockSE(chan, chan,3)\n self.block1_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block2_0 = ResidualBlockSE(3, chan,3)\n self.block2_1 = ResidualBlockSE(chan, chan,3)\n self.block2_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block3_0 = ResidualBlockSE(4, chan,3)\n self.block3_1 = ResidualBlockSE(chan, chan,3)\n self.block3_2 = ResidualBlockSE(chan, chan, 3)\n \n self.rnn = nn.GRU(4*chan, 128,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n\n self.fc0 = nn.Linear(256, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n \n x = torch.cat([x0,x1,x2,x3], dim=1)\n \n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x \nclass NetImuSE(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4, classes=18):\n super().__init__()\n \n self.block0_0 = ResidualBlockSE(4, chan,3)\n self.block0_1 = ResidualBlockSE(chan, chan,3)\n self.block0_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block1_0 = ResidualBlockSE(5, chan,3)\n self.block1_1 = ResidualBlockSE(chan, chan,3)\n self.block1_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block2_0 = ResidualBlockSE(3, chan,3)\n self.block2_1 = ResidualBlockSE(chan, chan,3)\n self.block2_2 = ResidualBlockSE(chan, chan, 3)\n\n self.block3_0 = ResidualBlockSE(4, chan,3)\n self.block3_1 = ResidualBlockSE(chan, chan,3)\n self.block3_2 = ResidualBlockSE(chan, chan, 3)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualBlockSE(4*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualBlockSE((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), classes)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n \n x = torch.cat([x0,x1,x2,x3], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x \n# https://www.kaggle.com/code/takoihiraokazu/cv-ensemble-sub-0607-1\nclass RnnModelImu(nn.Module):\n def __init__(\n self, dropout=0.2,\n input_numerical_size=16,\n numeraical_linear_size = 128,\n model_size = 128,\n linear_out = 128,\n out_size=18):\n super(RnnModelImu, self).__init__()\n self.numerical_linear = nn.Sequential(\n nn.Linear(input_numerical_size, numeraical_linear_size),\n nn.LayerNorm(numeraical_linear_size)\n )\n \n self.rnn = nn.GRU(numeraical_linear_size, model_size,\n num_layers = 2, \n bidirectional=True,\n batch_first=True)\n self.linear_out = nn.Sequential(\n nn.Linear(model_size*2, \n linear_out),\n nn.LayerNorm(linear_out),\n nn.Dropout(dropout),\n nn.Linear(linear_out, \n out_size))\n self._reinitialize()\n \n def _reinitialize(self):\n \"\"\"\n Tensorflow/Keras-like initialization\n \"\"\"\n for name, p in self.named_parameters():\n if 'rnn' in name:\n if 'weight_ih' in name:\n nn.init.xavier_uniform_(p.data)\n elif 'weight_hh' in name:\n nn.init.orthogonal_(p.data)\n elif 'bias_ih' in name:\n p.data.fill_(0)\n # Set forget-gate bias to 1\n n = p.size(0)\n p.data[(n // 4):(n // 2)].fill_(1)\n elif 'bias_hh' in name:\n p.data.fill_(0)\n \n def forward(self, numerical_array):\n numerical_array = numerical_array.permute(0, 2, 1)\n numerical_embedding = self.numerical_linear(numerical_array)\n output,_ = self.rnn(numerical_embedding)\n output = self.linear_out(output)\n return output[:,-30:,:].mean(axis=1)\n\n\nclass ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=1, dropout=0.0, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n out = self.act(out)\n \n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out \n\n\nclass NetAllResGru(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4):\n super().__init__()\n \n self.block0_0 = ResidualSECNNBlock(4, chan,3)\n self.block0_1 = ResidualSECNNBlock(chan, chan,3)\n self.block0_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block1_0 = ResidualSECNNBlock(5, chan,3)\n self.block1_1 = ResidualSECNNBlock(chan, chan,3)\n self.block1_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block2_0 = ResidualSECNNBlock(3, chan,3)\n self.block2_1 = ResidualSECNNBlock(chan, chan,3)\n self.block2_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block3_0 = ResidualSECNNBlock(4, chan,3)\n self.block3_1 = ResidualSECNNBlock(chan, chan,3)\n self.block3_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block4_0 = ResidualSECNNBlock(50, 2*chan,3)\n self.block4_1 = ResidualSECNNBlock(2*chan, 2*chan,5)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualSECNNBlock(6*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualSECNNBlock((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:16,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n x4 = self.block4_1(self.block4_0(x[:,16:,:]))\n \n x = torch.cat([x0,x1,x2, x3, x4], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x \n\n\nclass NetImuResGru(nn.Module):\n def __init__(self,n_blocks=2, chan=64, kern=5, kern_increase=4):\n super().__init__()\n \n self.block0_0 = ResidualSECNNBlock(4, chan,3)\n self.block0_1 = ResidualSECNNBlock(chan, chan,3)\n self.block0_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block1_0 = ResidualSECNNBlock(5, chan,3)\n self.block1_1 = ResidualSECNNBlock(chan, chan,3)\n self.block1_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block2_0 = ResidualSECNNBlock(3, chan,3)\n self.block2_1 = ResidualSECNNBlock(chan, chan,3)\n self.block2_2 = ResidualSECNNBlock(chan, chan, 3)\n\n self.block3_0 = ResidualSECNNBlock(4, chan,3)\n self.block3_1 = ResidualSECNNBlock(chan, chan,3)\n self.block3_2 = ResidualSECNNBlock(chan, chan, 3)\n \n self.layers = nn.ModuleList()\n for i in range(n_blocks+1):\n if i == 0:\n self.layers.append(ResidualSECNNBlock(4*chan, 4*chan,kern))\n else:\n self.layers.append(ResidualSECNNBlock((4*chan)*2**((i-1)//2), (4*chan)*2**(i//2),kern+kern_increase*i))\n self.pool = nn.MaxPool1d(2,2) \n self.fc0 = nn.Linear((4*chan)*2**(i//2), 18)\n \n def forward(self, x):\n in1 = torch.cat([x[:,:3,:], x[:,10:11,:]], dim=1)\n in2 = torch.cat([x[:,3:7,:], x[:,11:12,:]], dim=1)\n in3 = x[:,7:10,:]\n in4 = x[:,12:,:]\n x0 = self.block0_2(self.block0_1(self.block0_0(in1)))\n x1 = self.block1_2(self.block1_1(self.block1_0(in2)))\n x2 = self.block2_2(self.block2_1(self.block2_0(in3)))\n x3 = self.block3_2(self.block3_1(self.block3_0(in4)))\n \n x = torch.cat([x0,x1,x2,x3], dim=1)\n \n for i in range(len(self.layers)-1):\n x = self.pool(self.layers[i](x))\n x = self.layers[-1](x)\n x = torch.mean(x, dim=-1)\n x = self.fc0(x)\n return x ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.284834Z","iopub.execute_input":"2025-09-02T05:23:14.285059Z","iopub.status.idle":"2025-09-02T05:23:14.370011Z","shell.execute_reply.started":"2025-09-02T05:23:14.285043Z","shell.execute_reply":"2025-09-02T05:23:14.369299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def standard_scale_3d_transform(data, mean, std):\n return (data - mean) / std\n\ndef prep_data_tof3d(tof_arr,mean, std):\n tr_tof = standard_scale_3d_transform(tof_arr, mean, std)\n tr_tof = np.nan_to_num(tr_tof, 0)\n return tr_tof\n\nclass ResidualBlock3d(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size):\n super().__init__()\n self.conv1 = nn.Conv3d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn1 = nn.BatchNorm3d(num_features=out_channels)\n self.conv2 = nn.Conv3d(in_channels=out_channels, out_channels=out_channels, kernel_size=kernel_size, padding=\"same\")\n self.bn2 = nn.BatchNorm3d(num_features=out_channels)\n \n if in_channels != out_channels:\n self.res_layer = torch.nn.Conv3d(in_channels, out_channels, kernel_size=1, padding=\"same\")\n else:\n self.res_layer = None\n \n def forward(self, x):\n res = x\n x = F.relu(self.bn1(self.conv1(x)))\n x = F.relu(self.bn2(self.conv2(x)))\n if self.res_layer is None:\n x = torch.add(x, res)\n else: \n x = torch.add(x, self.res_layer(res)) # should the res layer have a relu? or batch norm?\n # TODO experiment with the residual block\n return x \n\n\nclass RNNet3d(nn.Module):\n def __init__(self, input_chan, chan=128):\n super().__init__()\n self.block0 = ResidualBlock3d(input_chan, chan, 3)\n self.block1 = ResidualBlock3d(chan, chan, 3)\n self.block2 = ResidualBlock3d(chan, chan, 3)\n\n self.rnn = nn.GRU(chan, 128,\n num_layers = 2, \n batch_first =True,\n bidirectional=True)\n \n self.pool = nn.MaxPool3d((1,2,2))\n self.fc0 = nn.Linear(256, 256)\n self.drop = nn.Dropout(0.2)\n self.fc1 = nn.Linear(256, 18)\n \n \n def forward(self, x):\n x = self.block0(x)\n x = self.pool(x)\n x = self.block1(x)\n x = self.pool(x)\n x = self.block2(x)\n x = self.pool(x)\n \n x = x.squeeze(dim=[3,4])\n x = x.permute(0, 2, 1)\n x,_ = self.rnn(x)\n x = self.drop(F.relu(self.fc0(x)))\n x = self.fc1(x)\n x = x[:,-30:,:].mean(axis=1)\n return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.3709Z","iopub.execute_input":"2025-09-02T05:23:14.371103Z","iopub.status.idle":"2025-09-02T05:23:14.385661Z","shell.execute_reply.started":"2025-09-02T05:23:14.371088Z","shell.execute_reply":"2025-09-02T05:23:14.384821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_folder_path = \"/kaggle/input/cmi-detect-behavior/twelfth_nn_submission/twelfth_nn_submission/\"\nimu_files_cnngru = glob.glob(os.path.join(model_folder_path, \"imuonly_cnngru128_itr*\"))\nimu_files_cnn = glob.glob(os.path.join(model_folder_path, \"imuonly_cnn64chan_itr*\"))\nimu_files_densegru = glob.glob(os.path.join(model_folder_path, \"imuonly_densegru_itr*\"))\nimu_files_cnngruSE = glob.glob(os.path.join(model_folder_path, \"imuonly_cnngru128SE_itr*\"))\nimu_files_cnnSE = glob.glob(os.path.join(model_folder_path, \"imuonly_cnn64chanSE_itr*\"))\n\nall_files_cnngru = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512_itr*\"))\nall_files_cnn = glob.glob(os.path.join(model_folder_path, \"alldata_cnn64chan_itr*\"))\nall_files_densegru = glob.glob(os.path.join(model_folder_path, \"alldata_densegru_itr*\"))\nall_files_cnngruSE = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512SE_itr*\"))\nall_files_cnnSE = glob.glob(os.path.join(model_folder_path, \"alldata_cnn64chanSE_itr*\"))\n#all_files_tofthm = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru128tofthm*\"))\n\nmodel_folder_path = \"/kaggle/input/cmi-detect-behavior/thirteenth_nn_submission/thirteenth_nn_submission/\"\nall_files_cnngru_diff = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512_diff*\"))\nall_files_cnn_diff = glob.glob(os.path.join(model_folder_path, \"alldata_cnn64chan_diff*\"))\nall_files_densegru_diff = glob.glob(os.path.join(model_folder_path, \"alldata_densegru_diff*\"))\nall_files_cnngruSE_diff = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512SE_diff*\"))\nall_files_cnnSE_diff = glob.glob(os.path.join(model_folder_path, \"alldata_cnn64chanSE_diff*\"))\n\nmodel_folder_path = \"/kaggle/input/cmi-detect-behavior/fourteenth_nn_submission/fourteenth_nn_submission/\"\nall_files_tof3dnet = glob.glob(os.path.join(model_folder_path, \"alldata_tof3dnet_itr*\"))\nall_files_tof3dnet_diff = glob.glob(os.path.join(model_folder_path, \"alldata_tof3dnet_diff*\"))\n\nmodel_folder_path = \"/kaggle/input/cmi-detect-behavior/seventeenth_nn_submission/seventeenth_nn_submission/\"\nimu_files_cnngru_resgru = glob.glob(os.path.join(model_folder_path, \"imuonly_cnngru128_resgru*\"))\nall_files_cnngru_resgru = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512_resgru*\"))\n\nmodel_folder_path = \"/kaggle/input/cmi-detect-orientation-models/nineteenth_nn_submission\"\nimu_files_orientation = glob.glob(os.path.join(model_folder_path, \"imuonly_cnn64chan*\"))\nall_files_orientation = glob.glob(os.path.join(model_folder_path, \"alldata_cnngru512*\"))\n\nif True:\n models_imu_cnngru = list()\n for mf in imu_files_cnngru:\n model = RNNetImu().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_imu_cnngru.append(model)\n print(len(models_imu_cnngru))\n \n models_imu_cnn = list()\n for mf in imu_files_cnn:\n model = NetImu().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_imu_cnn.append(model)\n print(len(models_imu_cnn))\n \n models_imu_densegru = list()\n for mf in imu_files_densegru:\n model = RnnModelImu().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_imu_densegru.append(model)\n print(len(models_imu_densegru))\n \n models_imu_cnngruSE = list()\n for mf in imu_files_cnngruSE:\n model = RNNetImuSE().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_imu_cnngruSE.append(model)\n print(len(models_imu_cnngruSE))\n \n models_imu_cnnSE = list()\n for mf in imu_files_cnnSE:\n model = NetImuSE().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_imu_cnnSE.append(model)\n print(len(models_imu_cnnSE))\n \n models_all_cnngru = list()\n for mf in all_files_cnngru:\n model = RNNetAllBig().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_cnngru.append(model)\n print(len(models_all_cnngru))\n \n models_all_cnn = list()\n for mf in all_files_cnn:\n model = NetAll().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_all_cnn.append(model)\n print(len(models_all_cnn))\n \n models_all_densegru = list()\n for mf in all_files_densegru:\n model = RnnModelAlldata().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_densegru.append(model)\n print(len(models_all_densegru))\n\n models_all_cnngruSE = list()\n for mf in all_files_cnngruSE:\n model = RNNetAllBigSE().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_cnngruSE.append(model)\n print(len(models_all_cnngruSE))\n \n models_all_cnnSE = list()\n for mf in all_files_cnnSE:\n model = NetAllSE().to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_all_cnnSE.append(model)\n print(len(models_all_cnnSE))\n \n # start adding diff\n models_all_cnngru_diff = list()\n for mf in all_files_cnngru_diff:\n model = RNNetAllBig(n_all_feat=100).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_cnngru_diff.append(model)\n print(len(models_all_cnngru_diff))\n \n models_all_cnn_diff = list()\n for mf in all_files_cnn_diff:\n model = NetAll(n_all_feat=100).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_all_cnn_diff.append(model)\n print(len(models_all_cnn_diff))\n \n models_all_densegru_diff = list()\n for mf in all_files_densegru_diff:\n model = RnnModelAlldata(input_numerical_size=116).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_densegru_diff.append(model)\n print(len(models_all_densegru_diff))\n \n models_all_cnngruSE_diff = list()\n for mf in all_files_cnngruSE_diff:\n model = RNNetAllBigSE(n_all_feat=100).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_cnngruSE_diff.append(model)\n print(len(models_all_cnngruSE_diff))\n \n models_all_cnnSE_diff = list()\n for mf in all_files_cnnSE_diff:\n model = NetAllSE(n_all_feat=100).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n models_all_cnnSE_diff.append(model)\n print(len(models_all_cnnSE_diff))\n \n models_all_tof3dnet = list()\n for mf in all_files_tof3dnet:\n model = RNNet3d(5).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_tof3dnet.append(model)\n print(len(models_all_tof3dnet))\n \n models_all_tof3dnet_diff = list()\n for mf in all_files_tof3dnet_diff:\n model = RNNet3d(10).to(device2)\n model_state = torch.load(mf, map_location=device2)\n model.load_state_dict(model_state)\n model.rnn.flatten_parameters()\n models_all_tof3dnet_diff.append(model)\n print(len(models_all_tof3dnet_diff))\n\n\nmodels_imu_cnngru_resgru = list()\nfor mf in imu_files_cnngru_resgru:\n model = NetImuResGru().to(device)\n model_state = torch.load(mf, map_location=device)\n model.load_state_dict(model_state)\n models_imu_cnngru_resgru.append(model)\nprint(len(models_imu_cnngru_resgru))\n\nmodels_all_cnngru_resgru = list()\nfor mf in all_files_cnngru_resgru:\n model = NetAllResGru().to(device)\n model_state = torch.load(mf, map_location=device)\n model.load_state_dict(model_state)\n models_all_cnngru_resgru.append(model)\nprint(len(models_all_cnngru_resgru))\n\nmodels_imu_orientation = list()\nfor mf in imu_files_orientation:\n model = NetImuSE(classes=4).to(device)\n model_state = torch.load(mf, map_location=device)\n model.load_state_dict(model_state)\n models_imu_orientation.append(model)\nprint(len(models_imu_orientation))\n\nmodels_all_orientation = list()\nfor mf in all_files_orientation:\n model = RNNetAllBig(classes=4).to(device)\n model_state = torch.load(mf, map_location=device)\n model.load_state_dict(model_state)\n models_all_orientation.append(model)\nprint(len(models_all_orientation))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:23:14.386397Z","iopub.execute_input":"2025-09-02T05:23:14.386647Z","iopub.status.idle":"2025-09-02T05:26:24.758783Z","shell.execute_reply.started":"2025-09-02T05:23:14.386627Z","shell.execute_reply":"2025-09-02T05:26:24.758106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with open(\"/kaggle/input/cmi-detect-behavior/twelfth_nn_submission/twelfth_nn_submission/imuonly_scaler.pkl\", \"rb\") as f:\n scaler_imu = load(f)\n\nwith open(\"/kaggle/input/cmi-detect-behavior/twelfth_nn_submission/twelfth_nn_submission/alldata_scaler.pkl\", \"rb\") as f:\n scaler_all = load(f)\n\nwith open(\"/kaggle/input/cmi-detect-behavior/thirteenth_nn_submission/thirteenth_nn_submission/alldata_diff_scaler.pkl\", \"rb\") as f:\n scaler_all_diff = load(f)\n\nwith open(\"/kaggle/input/cmi-detect-orientation-models/nineteenth_nn_submission/orientation_encoder.pkl\", \"rb\") as f:\n orientation_encoder = load(f)\n\ntof_mean = np.load(\"/kaggle/input/cmi-detect-behavior/fourteenth_nn_submission/fourteenth_nn_submission/tof_mean.npy\")\ntof_std = np.load(\"/kaggle/input/cmi-detect-behavior/fourteenth_nn_submission/fourteenth_nn_submission/tof_std.npy\")\n\ntof_diff_mean = np.load(\"/kaggle/input/cmi-detect-behavior/fourteenth_nn_submission/fourteenth_nn_submission/tof_diff_mean.npy\")\ntof_diff_std = np.load(\"/kaggle/input/cmi-detect-behavior/fourteenth_nn_submission/fourteenth_nn_submission/tof_diff_std.npy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.759572Z","iopub.execute_input":"2025-09-02T05:26:24.759797Z","iopub.status.idle":"2025-09-02T05:26:24.811643Z","shell.execute_reply.started":"2025-09-02T05:26:24.75978Z","shell.execute_reply":"2025-09-02T05:26:24.811112Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tof_to_array(df, sensor, t):\n l = list()\n for i in range(8):\n ll = list()\n for j in range(8):\n ll.append(df[t,f\"tof_{sensor}_v{i*8+j}\"])\n l.append(ll)\n return np.array(l)\n\ndef tofdiff_to_array(df, sensor, t):\n l = list()\n for i in range(8):\n ll = list()\n for j in range(8):\n ll.append(df[t,f\"tof_{sensor}_v{i*8+j}_diff\"])\n l.append(ll)\n return np.array(l)\n\ndef prep_tof(df, mean, std):\n tof_arr = np.zeros((1, 5, 128, 8, 8))\n start = max(df.shape[0]-128, 0)\n offset = max(128-df.shape[0], 0)\n for ee, i in enumerate(range(start, df.shape[0])):\n for j in range(1,6):\n tof_image = tof_to_array(df, j, i)\n tof_arr[0,j-1,offset+ee,:,:] = tof_image\n return prep_data_tof3d(tof_arr, mean, std)\n\ndef prep_tof_diff(df, mean, std):\n df = df.with_columns(pl.all().replace(-1, None))\n for i in [f\"tof_{x}_v{y}\" for x in range(1,6) for y in range(64)]:\n df = df.with_columns(pl.col(i).diff().alias(i+\"_diff\"))\n tof_diff_arr = np.zeros((1, 10, 128, 8, 8))\n \n start = max(df.shape[0]-128, 0)\n offset = max(128-df.shape[0], 0)\n for ee, i in enumerate(range(start, df.shape[0])):\n for j in range(1,6):\n tof_image = tof_to_array(df, j, i)\n tof_diff_arr[0,j-1,offset+ee,:,:] = tof_image\n for j in range(1,6):\n tof_image = tofdiff_to_array(df, j, i)\n tof_diff_arr[0,5+j-1,offset+ee,:,:] = tof_image\n return prep_data_tof3d(tof_diff_arr, mean, std)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.812364Z","iopub.execute_input":"2025-09-02T05:26:24.812572Z","iopub.status.idle":"2025-09-02T05:26:24.821662Z","shell.execute_reply.started":"2025-09-02T05:26:24.812557Z","shell.execute_reply":"2025-09-02T05:26:24.821037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_checker():\n count = dict()\n def checker(subj_id, acc_y):\n nonlocal count\n if subj_id in count:\n count[subj_id].append(acc_y)\n else:\n count[subj_id] = [acc_y]\n a = pl.Series(count[subj_id])\n m = a.mean()\n l = a.shape[0]\n if m > 0.5 and l > 70:\n return True\n else:\n return False\n return checker","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.822325Z","iopub.execute_input":"2025-09-02T05:26:24.822542Z","iopub.status.idle":"2025-09-02T05:26:24.83479Z","shell.execute_reply.started":"2025-09-02T05:26:24.822526Z","shell.execute_reply":"2025-09-02T05:26:24.834027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"c = make_checker()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.835579Z","iopub.execute_input":"2025-09-02T05:26:24.836213Z","iopub.status.idle":"2025-09-02T05:26:24.845969Z","shell.execute_reply.started":"2025-09-02T05:26:24.836195Z","shell.execute_reply":"2025-09-02T05:26:24.845469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_inner(inputs, models):\n for e, model in enumerate(models):\n with torch.no_grad():\n model.eval()\n outputs = model(inputs)\n if e == 0:\n predicted = outputs #.to(\"cpu\").numpy()\n else:\n predicted += outputs #.to(\"cpu\").numpy()\n return predicted","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.846679Z","iopub.execute_input":"2025-09-02T05:26:24.846877Z","iopub.status.idle":"2025-09-02T05:26:24.858279Z","shell.execute_reply.started":"2025-09-02T05:26:24.846855Z","shell.execute_reply":"2025-09-02T05:26:24.857767Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Predicting","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# zhou","metadata":{}},{"cell_type":"code","source":"!pip install --no-deps /kaggle/input/scikit-learn-1-6-1/scikit_learn-1.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\n\n#!/usr/bin/env python\n# coding: utf-8\n\n# # Importable version of https://www.kaggle.com/code/metric/cmi-2025\n\n# In[ ]:\n\n\n\"\"\"\nHierarchical macro F1 metric for the CMI 2025 Challenge.\n\nThis script defines a single entry point `score(solution, submission, row_id_column_name)`\nthat the Kaggle metrics orchestrator will call.\nIt performs validation on submission IDs and computes a combined binary & multiclass F1 score.\n\"\"\"\n\nimport pandas as pd\nfrom sklearn.metrics import f1_score\n\n\nclass ParticipantVisibleError(Exception):\n \"\"\"Errors raised here will be shown directly to the competitor.\"\"\"\n pass\n\n\nclass CompetitionMetric:\n \"\"\"Hierarchical macro F1 for the CMI 2025 challenge.\"\"\"\n def __init__(self):\n self.target_gestures = [\n 'Above ear - pull hair',\n 'Cheek - pinch skin',\n 'Eyebrow - pull hair',\n 'Eyelash - pull hair',\n 'Forehead - pull hairline',\n 'Forehead - scratch',\n 'Neck - pinch skin',\n 'Neck - scratch',\n ]\n self.non_target_gestures = [\n 'Write name on leg',\n 'Wave hello',\n 'Glasses on/off',\n 'Text on phone',\n 'Write name in air',\n 'Feel around in tray and pull out an object',\n 'Scratch knee/leg skin',\n 'Pull air toward your face',\n 'Drink from bottle/cup',\n 'Pinch knee/leg skin'\n ]\n self.all_classes = self.target_gestures + self.non_target_gestures\n\n def calculate_hierarchical_f1(\n self,\n sol: pd.DataFrame,\n sub: pd.DataFrame\n ) -> float:\n\n # Validate gestures\n invalid_types = {i for i in sub['gesture'].unique() if i not in self.all_classes}\n if invalid_types:\n raise ParticipantVisibleError(\n f\"Invalid gesture values in submission: {invalid_types}\"\n )\n\n # Compute binary F1 (Target vs Non-Target)\n y_true_bin = sol['gesture'].isin(self.target_gestures).values\n y_pred_bin = sub['gesture'].isin(self.target_gestures).values\n f1_binary = f1_score(\n y_true_bin,\n y_pred_bin,\n pos_label=True,\n zero_division=0,\n average='binary'\n )\n\n # Build multi-class labels for gestures\n y_true_mc = sol['gesture'].apply(lambda x: x if x in self.target_gestures else 'non_target')\n y_pred_mc = sub['gesture'].apply(lambda x: x if x in self.target_gestures else 'non_target')\n\n # Compute macro F1 over all gesture classes\n f1_macro = f1_score(\n y_true_mc,\n y_pred_mc,\n average='macro',\n zero_division=0\n )\n\n return 0.5 * f1_binary + 0.5 * f1_macro\n\n\ndef score(\n solution: pd.DataFrame,\n submission: pd.DataFrame,\n row_id_column_name: str\n) -> float:\n \"\"\"\n Compute hierarchical macro F1 for the CMI 2025 challenge.\n\n Expected input:\n - solution and submission as pandas.DataFrame\n - Column 'sequence_id': unique identifier for each sequence\n - 'gesture': one of the eight target gestures or \"Non-Target\"\n\n This metric averages:\n 1. Binary F1 on SequenceType (Target vs Non-Target)\n 2. Macro F1 on gesture (mapping non-targets to \"Non-Target\")\n\n Raises ParticipantVisibleError for invalid submissions,\n including invalid SequenceType or gesture values.\n\n\n Examples\n --------\n >>> import pandas as pd\n >>> row_id_column_name = \"id\"\n >>> solution = pd.DataFrame({'id': range(4), 'gesture': ['Eyebrow - pull hair']*4})\n >>> submission = pd.DataFrame({'id': range(4), 'gesture': ['Forehead - pull hairline']*4})\n >>> score(solution, submission, row_id_column_name=row_id_column_name)\n 0.5\n >>> submission = pd.DataFrame({'id': range(4), 'gesture': ['Text on phone']*4})\n >>> score(solution, submission, row_id_column_name=row_id_column_name)\n 0.0\n >>> score(solution, solution, row_id_column_name=row_id_column_name)\n 1.0\n \"\"\"\n # Validate required columns\n for col in (row_id_column_name, 'gesture'):\n if col not in solution.columns:\n raise ParticipantVisibleError(f\"Solution file missing required column: '{col}'\")\n if col not in submission.columns:\n raise ParticipantVisibleError(f\"Submission file missing required column: '{col}'\")\n\n metric = CompetitionMetric()\n return metric.calculate_hierarchical_f1(solution, submission)\n\n\nimport torch\nimport torch.distributed as dist\n\n\ndef zeropower_via_newtonschulz5(G, steps: int):\n \"\"\"\n Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a\n quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose\n of minimizing steps, it turns out to be empirically effective to keep increasing the slope at\n zero even beyond the point where the iteration no longer converges all the way to one everywhere\n on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T\n where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model\n performance at all relative to UV^T, where USV^T = G is the SVD.\n \"\"\"\n assert G.ndim >= 2 # batched Muon implementation by @scottjmaddox, and put into practice in the record by @YouJiacheng\n a, b, c = (3.4445, -4.7750, 2.0315)\n X = G.bfloat16()\n if G.size(-2) > G.size(-1):\n X = X.mT\n\n # Ensure spectral norm is at most 1\n X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)\n # Perform the NS iterations\n for _ in range(steps):\n A = X @ X.mT\n B = b * A + c * A @ A # quintic computation strategy adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng\n X = a * X + B @ X\n \n if G.size(-2) > G.size(-1):\n X = X.mT\n return X\n\n\ndef muon_update(grad, momentum, beta=0.95, ns_steps=5, nesterov=True):\n momentum.lerp_(grad, 1 - beta)\n update = grad.lerp_(momentum, beta) if nesterov else momentum\n if update.ndim == 4: # for the case of conv filters\n update = update.view(len(update), -1)\n update = zeropower_via_newtonschulz5(update, steps=ns_steps)\n update *= max(1, grad.size(-2) / grad.size(-1))**0.5\n return update\n\n\nclass Muon(torch.optim.Optimizer):\n \"\"\"\n Muon - MomentUm Orthogonalized by Newton-schulz\n\n https://kellerjordan.github.io/posts/muon/\n\n Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-\n processing step, in which each 2D parameter's update is replaced with the nearest orthogonal\n matrix. For efficient orthogonalization we use a Newton-Schulz iteration, which has the\n advantage that it can be stably run in bfloat16 on the GPU.\n\n Muon should only be used for hidden weight layers. The input embedding, final output layer,\n and any internal gains or biases should be optimized using a standard method such as AdamW.\n Hidden convolutional weights can be trained using Muon by viewing them as 2D and then\n collapsing their last 3 dimensions.\n\n Arguments:\n lr: The learning rate, in units of spectral norm per update.\n weight_decay: The AdamW-style weight decay.\n momentum: The momentum. A value of 0.95 here is usually fine.\n \"\"\"\n def __init__(self, params, lr=0.02, weight_decay=0, momentum=0.95):\n defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum)\n assert isinstance(params, list) and len(params) >= 1 and isinstance(params[0], torch.nn.Parameter)\n params = sorted(params, key=lambda x: x.size(), reverse=True)\n super().__init__(params, defaults)\n\n @torch.no_grad()\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n params = group[\"params\"]\n params_pad = params + [torch.empty_like(params[-1])] * (dist.get_world_size() - len(params) % dist.get_world_size())\n for base_i in range(len(params))[::dist.get_world_size()]:\n if base_i + dist.get_rank() < len(params):\n p = params[base_i + dist.get_rank()]\n state = self.state[p]\n if len(state) == 0:\n state[\"momentum_buffer\"] = torch.zeros_like(p)\n update = muon_update(p.grad, state[\"momentum_buffer\"], beta=group[\"momentum\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update.reshape(p.shape), alpha=-group[\"lr\"])\n dist.all_gather(params_pad[base_i:base_i + dist.get_world_size()], params_pad[base_i + dist.get_rank()])\n\n return loss\n\n\nclass SingleDeviceMuon(torch.optim.Optimizer):\n \"\"\"\n Muon variant for usage in non-distributed settings.\n \"\"\"\n def __init__(self, params, lr=0.02, weight_decay=0, momentum=0.95):\n defaults = dict(lr=lr, weight_decay=weight_decay, momentum=momentum)\n super().__init__(params, defaults)\n\n @torch.no_grad()\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n for p in group[\"params\"]:\n state = self.state[p]\n if len(state) == 0:\n state[\"momentum_buffer\"] = torch.zeros_like(p)\n update = muon_update(p.grad, state[\"momentum_buffer\"], beta=group[\"momentum\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update.reshape(p.shape), alpha=-group[\"lr\"])\n\n return loss\n\n\ndef adam_update(grad, buf1, buf2, step, betas, eps):\n buf1.lerp_(grad, 1 - betas[0])\n buf2.lerp_(grad.square(), 1 - betas[1])\n buf1c = buf1 / (1 - betas[0]**step)\n buf2c = buf2 / (1 - betas[1]**step)\n return buf1c / (buf2c.sqrt() + eps)\n\n\nclass MuonWithAuxAdam(torch.optim.Optimizer):\n \"\"\"\n Distributed Muon variant that can be used for all parameters in the network, since it runs an\n internal AdamW for the parameters that are not compatible with Muon. The user must manually\n specify which parameters shall be optimized with Muon and which with Adam by passing in a\n list of param_groups with the `use_muon` flag set.\n\n The point of this class is to allow the user to have a single optimizer in their code, rather\n than having both a Muon and an Adam which each need to be stepped.\n\n You can see an example usage below:\n\n https://github.com/KellerJordan/modded-nanogpt/blob/master/records/052525_MuonWithAuxAdamExample/b01550f9-03d8-4a9c-86fe-4ab434f1c5e0.txt#L470\n ```\n hidden_matrix_params = [p for n, p in model.blocks.named_parameters() if p.ndim >= 2 and \"embed\" not in n]\n embed_params = [p for n, p in model.named_parameters() if \"embed\" in n]\n scalar_params = [p for p in model.parameters() if p.ndim < 2]\n head_params = [model.lm_head.weight]\n\n from muon import MuonWithAuxAdam\n adam_groups = [dict(params=head_params, lr=0.22), dict(params=embed_params, lr=0.6), dict(params=scalar_params, lr=0.04)]\n adam_groups = [dict(**g, betas=(0.8, 0.95), eps=1e-10, use_muon=False) for g in adam_groups]\n muon_group = dict(params=hidden_matrix_params, lr=0.05, momentum=0.95, use_muon=True)\n param_groups = [*adam_groups, muon_group]\n optimizer = MuonWithAuxAdam(param_groups)\n ```\n \"\"\"\n def __init__(self, param_groups):\n for group in param_groups:\n assert \"use_muon\" in group\n if group[\"use_muon\"]:\n group[\"params\"] = sorted(group[\"params\"], key=lambda x: x.size(), reverse=True)\n # defaults\n group[\"lr\"] = group.get(\"lr\", 0.02)\n group[\"momentum\"] = group.get(\"momentum\", 0.95)\n group[\"weight_decay\"] = group.get(\"weight_decay\", 0)\n assert set(group.keys()) == set([\"params\", \"lr\", \"momentum\", \"weight_decay\", \"use_muon\"])\n else:\n # defaults\n group[\"lr\"] = group.get(\"lr\", 3e-4)\n group[\"betas\"] = group.get(\"betas\", (0.9, 0.95))\n group[\"eps\"] = group.get(\"eps\", 1e-10)\n group[\"weight_decay\"] = group.get(\"weight_decay\", 0)\n assert set(group.keys()) == set([\"params\", \"lr\", \"betas\", \"eps\", \"weight_decay\", \"use_muon\"])\n super().__init__(param_groups, dict())\n\n @torch.no_grad()\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n if group[\"use_muon\"]:\n params = group[\"params\"]\n params_pad = params + [torch.empty_like(params[-1])] * (dist.get_world_size() - len(params) % dist.get_world_size())\n for base_i in range(len(params))[::dist.get_world_size()]:\n if base_i + dist.get_rank() < len(params):\n p = params[base_i + dist.get_rank()]\n state = self.state[p]\n if len(state) == 0:\n state[\"momentum_buffer\"] = torch.zeros_like(p)\n update = muon_update(p.grad, state[\"momentum_buffer\"], beta=group[\"momentum\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update.reshape(p.shape), alpha=-group[\"lr\"])\n dist.all_gather(params_pad[base_i:base_i + dist.get_world_size()], params_pad[base_i + dist.get_rank()])\n else:\n for p in group[\"params\"]:\n state = self.state[p]\n if len(state) == 0:\n state[\"exp_avg\"] = torch.zeros_like(p)\n state[\"exp_avg_sq\"] = torch.zeros_like(p)\n state[\"step\"] = 0\n state[\"step\"] += 1\n update = adam_update(p.grad, state[\"exp_avg\"], state[\"exp_avg_sq\"],\n state[\"step\"], group[\"betas\"], group[\"eps\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update, alpha=-group[\"lr\"])\n\n return loss\n\n\nclass SingleDeviceMuonWithAuxAdam(torch.optim.Optimizer):\n \"\"\"\n Non-distributed variant of MuonWithAuxAdam.\n \"\"\"\n def __init__(self, param_groups):\n for group in param_groups:\n assert \"use_muon\" in group\n if group[\"use_muon\"]:\n # defaults\n group[\"lr\"] = group.get(\"lr\", 0.02)\n group[\"momentum\"] = group.get(\"momentum\", 0.95)\n group[\"weight_decay\"] = group.get(\"weight_decay\", 0)\n assert set(group.keys()) == set([\"params\", \"lr\", \"momentum\", \"weight_decay\", \"use_muon\"])\n else:\n # defaults\n group[\"lr\"] = group.get(\"lr\", 3e-4)\n group[\"betas\"] = group.get(\"betas\", (0.9, 0.95))\n group[\"eps\"] = group.get(\"eps\", 1e-10)\n group[\"weight_decay\"] = group.get(\"weight_decay\", 0)\n assert set(group.keys()) == set([\"params\", \"lr\", \"betas\", \"eps\", \"weight_decay\", \"use_muon\"])\n super().__init__(param_groups, dict())\n\n @torch.no_grad()\n def step(self, closure=None):\n\n loss = None\n if closure is not None:\n with torch.enable_grad():\n loss = closure()\n\n for group in self.param_groups:\n if group[\"use_muon\"]:\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n \n state = self.state[p]\n if len(state) == 0:\n state[\"momentum_buffer\"] = torch.zeros_like(p)\n update = muon_update(p.grad, state[\"momentum_buffer\"], beta=group[\"momentum\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update.reshape(p.shape), alpha=-group[\"lr\"])\n else:\n for p in group[\"params\"]:\n if p.grad is None:\n continue\n state = self.state[p]\n if len(state) == 0:\n state[\"exp_avg\"] = torch.zeros_like(p)\n state[\"exp_avg_sq\"] = torch.zeros_like(p)\n state[\"step\"] = 0\n state[\"step\"] += 1\n update = adam_update(p.grad, state[\"exp_avg\"], state[\"exp_avg_sq\"],\n state[\"step\"], group[\"betas\"], group[\"eps\"])\n p.mul_(1 - group[\"lr\"] * group[\"weight_decay\"])\n p.add_(update, alpha=-group[\"lr\"])\n\n return loss\n\nfrom torch.optim.lr_scheduler import _LRScheduler\nclass ConstantCosineLR(_LRScheduler):\n \"\"\"\n Constant learning rate followed by CosineAnnealing.\n \"\"\"\n def __init__(\n self, \n optimizer,\n total_steps, \n pct_cosine, \n last_epoch=-1,\n ):\n self.total_steps = total_steps\n self.milestone = int(total_steps * (pct_cosine))\n self.cosine_steps = max(total_steps - self.milestone, 1)\n self.min_lr = 0\n super().__init__(optimizer, last_epoch)\n\n def get_lr(self):\n step = self.last_epoch + 1\n if step <= self.milestone:\n factor = 1.\n else:\n s = step - self.milestone\n factor = 0.5 * (1 + math.cos(math.pi * s / self.cosine_steps))\n return [lr * factor for lr in self.base_lrs]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:24.858983Z","iopub.execute_input":"2025-09-02T05:26:24.859192Z","iopub.status.idle":"2025-09-02T05:26:28.750272Z","shell.execute_reply.started":"2025-09-02T05:26:24.85917Z","shell.execute_reply":"2025-09-02T05:26:28.74955Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:28.751371Z","iopub.execute_input":"2025-09-02T05:26:28.751758Z","iopub.status.idle":"2025-09-02T05:26:28.755746Z","shell.execute_reply.started":"2025-09-02T05:26:28.751733Z","shell.execute_reply":"2025-09-02T05:26:28.755078Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v1:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\"):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(\"./\") \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(\"./\") \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(\"./\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n self.VALIDATION = False\n self.SEED = 42\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, \n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n out += shortcut\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (9 + 4 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 256\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:]) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:]) * mask_all.transpose(1, 2)\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def pad_sequences_torch(self, seq, maxlen, padding='post', truncating='post', value=0.0):\n \n if seq.shape[0] >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - seq.shape[0]\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n return seq \n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n idlistall.append(id_list_valid)\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA\n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n \n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, X_imuonly, y = X.float().to(device), X_imuonly.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n \n optimizer.zero_grad()\n \n logits, logits2, logits3 = model(X)\n # logits_, logits2_, logits3_ = model(X_imuonly)\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2, clipval, 5) * 0.4\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.4\n \n # loss += clipped_cross_entropy(logits_, y, clipval, 18) * 0.1\n # loss += clipped_cross_entropy(logits2_, y2, clipval, 5) * 0.4 * 0.1\n # loss += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.4 * 0.1\n \n # loss += 1 * (criterion_rdrop(logits, logits_) + criterion_rdrop(logits2, logits2_) + criterion_rdrop(logits3, logits3_))\n \n loss.backward()\n \n optimizer.step()\n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n \n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds, val_targets = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_all, val_targets_all = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n \n logits = model(X)\n logits_imuonly = model(X_imuonly)\n \n val_preds.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n\n val_preds_logits.append(logits.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_preds_all.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_all.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_targets.append(y.argmax(dim=1).cpu().numpy())\n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n \n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n \n loss = F.cross_entropy(logits, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_split = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_all)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_all)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ALL : ', round(val_acc, 4), '| SPLIT : ', round(val_acc_split, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len\n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n df_linear_accel = pd.concat(linear_accel_list)\n df = pd.concat([df, df_linear_accel], axis=1)\n \n angular_vel_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n df_angular_vel = pd.concat(angular_vel_list)\n df = pd.concat([df, df_angular_vel], axis=1)\n \n angular_distance_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in tqdm(range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n if self.tof_mode == -1:\n for mode in [2, 4, 8, 16, 32]:\n region_size = 64 // mode\n for r in tqdm(range(mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{mode}_{i}_region_{r}_max': region_data.max(axis=1)\n })\n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(5) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio=0.): \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n for model in self.models:\n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).to(self.device)\n \n model.eval()\n p = model(x)\n if outputs is None: outputs = p\n else: outputs += p\n outputs /= len(self.models)\n \n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:28.757594Z","iopub.execute_input":"2025-09-02T05:26:28.757882Z","iopub.status.idle":"2025-09-02T05:26:29.020286Z","shell.execute_reply.started":"2025-09-02T05:26:28.757865Z","shell.execute_reply":"2025-09-02T05:26:29.019709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v1_1 = model_zhou_v1('/kaggle/input/cmi3v35')\n\n# code for validation score prediction\nif VALIDATION:\n # use scikit-learn 1.6.1 for groupkfold split with seed\n MMM = model_zhou_inference_v1_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_1_1, pred_imuonly_1_1, targets, idlist_1_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_1_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_1_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.020956Z","iopub.execute_input":"2025-09-02T05:26:29.021742Z","iopub.status.idle":"2025-09-02T05:26:29.02761Z","shell.execute_reply.started":"2025-09-02T05:26:29.021723Z","shell.execute_reply":"2025-09-02T05:26:29.026736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v1_2 = model_zhou_v1('/kaggle/input/cmi3v38')\n\n# code for validation score prediction\nif VALIDATION:\n # use scikit-learn 1.6.1 for groupkfold split with seed\n MMM = model_zhou_inference_v1_2\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_1_2, pred_imuonly_1_2, targets, idlist_1_2 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_1_2.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_1_2.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.028547Z","iopub.execute_input":"2025-09-02T05:26:29.028828Z","iopub.status.idle":"2025-09-02T05:26:29.044461Z","shell.execute_reply.started":"2025-09-02T05:26:29.02881Z","shell.execute_reply":"2025-09-02T05:26:29.043855Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v14:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 8\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n out += shortcut\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (9 + 4 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 256\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:]) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:]) * mask_all.transpose(1, 2)\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA\n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n \n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n\n \n X[BS//8:, :, 14:] = 0.0\n \n \n X, X_imuonly, y = X.float().to(device), X_imuonly.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n \n optimizer.zero_grad()\n \n LEN = 170 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n \n loss.backward()\n \n optimizer.step()\n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n \n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds, val_targets = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_all, val_targets_all = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n \n logits = model(X)\n logits_imuonly = model(X_imuonly)\n \n val_preds.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n\n val_preds_logits.append(logits.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_preds_all.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_all.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_targets.append(y.argmax(dim=1).cpu().numpy())\n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n \n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n \n loss = F.cross_entropy(logits, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_split = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_all)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_all)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ALL : ', round(val_acc, 4), '| SPLIT : ', round(val_acc_split, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n if self.tof_mode == -1:\n for mode in [2, 4, 8, 16, 32]:\n region_size = 64 // mode\n for r in (range(mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{mode}_{i}_region_{r}_max': region_data.max(axis=1)\n })\n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p = model(x)\n if outputs is None: outputs = p\n else: outputs += p\n outputs /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.045598Z","iopub.execute_input":"2025-09-02T05:26:29.04583Z","iopub.status.idle":"2025-09-02T05:26:29.175633Z","shell.execute_reply.started":"2025-09-02T05:26:29.045815Z","shell.execute_reply":"2025-09-02T05:26:29.17499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v14_1 = model_zhou_v14('/kaggle/input/cmi3v59/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v14_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_14_1, pred_imuonly_14_1, targets, idlist_14_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_14_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_14_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.176419Z","iopub.execute_input":"2025-09-02T05:26:29.176711Z","iopub.status.idle":"2025-09-02T05:26:29.198102Z","shell.execute_reply.started":"2025-09-02T05:26:29.176694Z","shell.execute_reply":"2025-09-02T05:26:29.197577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v14_2 = model_zhou_v14('/kaggle/input/cmi3v59/666', 666, './666')\nif VALIDATION:\n MMM = model_zhou_inference_v14_2\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_14_2, pred_imuonly_14_2, targets, idlist_14_2 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_14_2.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_14_2.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.199066Z","iopub.execute_input":"2025-09-02T05:26:29.199338Z","iopub.status.idle":"2025-09-02T05:26:29.207926Z","shell.execute_reply.started":"2025-09-02T05:26:29.199315Z","shell.execute_reply":"2025-09-02T05:26:29.207274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v14_3 = model_zhou_v14('/kaggle/input/cmi3v59/8888', 8888, './8888')\nif VALIDATION:\n MMM = model_zhou_inference_v14_3\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_14_3, pred_imuonly_14_3, targets, idlist_14_3 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_14_3.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_14_3.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.208594Z","iopub.execute_input":"2025-09-02T05:26:29.2088Z","iopub.status.idle":"2025-09-02T05:26:29.225343Z","shell.execute_reply.started":"2025-09-02T05:26:29.208781Z","shell.execute_reply":"2025-09-02T05:26:29.224524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v15:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 8\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n out += shortcut\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (9 + 4 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.GRU(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:]) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:]) * mask_all.transpose(1, 2)\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA\n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n\n X[:BS//8,:,14:] = 0.\n \n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, X_imuonly, y = X.float().to(device), X_imuonly.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n \n optimizer.zero_grad()\n \n LEN = 170 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.8\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.8\n \n loss.backward()\n \n optimizer.step()\n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n \n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds, val_targets = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_all, val_targets_all = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n \n logits = model(X)\n logits_imuonly = model(X_imuonly)\n \n val_preds.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n\n val_preds_logits.append(logits.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_preds_all.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_all.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_targets.append(y.argmax(dim=1).cpu().numpy())\n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n \n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n \n loss = F.cross_entropy(logits, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_split = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_all)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_all)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ALL : ', round(val_acc, 4), '| SPLIT : ', round(val_acc_split, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n if self.tof_mode == -1:\n for mode in [2, 4, 8, 16, 32]:\n region_size = 64 // mode\n for r in (range(mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{mode}_{i}_region_{r}_max': region_data.max(axis=1)\n })\n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p = model(x)\n if outputs is None: outputs = p\n else: outputs += p\n outputs /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.22683Z","iopub.execute_input":"2025-09-02T05:26:29.227092Z","iopub.status.idle":"2025-09-02T05:26:29.351763Z","shell.execute_reply.started":"2025-09-02T05:26:29.227077Z","shell.execute_reply":"2025-09-02T05:26:29.350915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v15_1 = model_zhou_v15('/kaggle/input/cmi3v58/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v15_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_15_1, pred_imuonly_15_1, targets, idlist_15_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_15_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_15_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.352543Z","iopub.execute_input":"2025-09-02T05:26:29.352737Z","iopub.status.idle":"2025-09-02T05:26:29.358747Z","shell.execute_reply.started":"2025-09-02T05:26:29.352724Z","shell.execute_reply":"2025-09-02T05:26:29.357958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v15_2 = model_zhou_v15('/kaggle/input/cmi3v58/6665252', 6665252, './6665252')\nif VALIDATION:\n MMM = model_zhou_inference_v15_2\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_15_2, pred_imuonly_15_2, targets, idlist_15_2 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_15_2.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_15_2.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.359505Z","iopub.execute_input":"2025-09-02T05:26:29.360189Z","iopub.status.idle":"2025-09-02T05:26:29.372955Z","shell.execute_reply.started":"2025-09-02T05:26:29.360154Z","shell.execute_reply":"2025-09-02T05:26:29.372299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v15_3 = model_zhou_v15('/kaggle/input/cmi3v58/88885252', 88885252, './88885252')\nif VALIDATION:\n MMM = model_zhou_inference_v15_3\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, 5, 6, 7]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_15_3, pred_imuonly_15_3, targets, idlist_15_3 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_15_3.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_15_3.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.373629Z","iopub.execute_input":"2025-09-02T05:26:29.373881Z","iopub.status.idle":"2025-09-02T05:26:29.387827Z","shell.execute_reply.started":"2025-09-02T05:26:29.373861Z","shell.execute_reply":"2025-09-02T05:26:29.387154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v16:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 150\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n out += shortcut\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (9 + 4 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 256\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:]) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:]) * mask_all.transpose(1, 2)\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n # group_list = []\n # for _, group in tqdm(df.groupby('sequence_id')):\n # index_old = group.index\n # group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n # group.index = index_old\n \n # group_list.append(group)\n # df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA\n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n\n # X[:BS//8,:,14:] = 0.\n \n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, X_imuonly, y = X.float().to(device), X_imuonly.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n \n optimizer.zero_grad()\n \n LEN = 100 + np.random.randint(100)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.8\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.8\n \n loss.backward()\n \n optimizer.step()\n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n \n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds, val_targets = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_all, val_targets_all = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n \n logits = model(X)\n logits_imuonly = model(X_imuonly)\n \n val_preds.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n\n val_preds_logits.append(logits.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_preds_all.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_all.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_targets.append(y.argmax(dim=1).cpu().numpy())\n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n \n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n \n loss = F.cross_entropy(logits, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_split = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_all)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_all)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ALL : ', round(val_acc, 4), '| SPLIT : ', round(val_acc_split, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n if self.tof_mode == -1:\n for mode in [2, 4, 8, 16, 32]:\n region_size = 64 // mode\n for r in (range(mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{mode}_{i}_region_{r}_max': region_data.max(axis=1)\n })\n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n # sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p = model(x)\n if outputs is None: outputs = p\n else: outputs += p\n outputs /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.388941Z","iopub.execute_input":"2025-09-02T05:26:29.389132Z","iopub.status.idle":"2025-09-02T05:26:29.514926Z","shell.execute_reply.started":"2025-09-02T05:26:29.389112Z","shell.execute_reply":"2025-09-02T05:26:29.514096Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v16_1 = model_zhou_v16('/kaggle/input/cmi3v61', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v16_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_16_1, pred_imuonly_16_1, targets, idlist_16_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_16_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_16_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.515923Z","iopub.execute_input":"2025-09-02T05:26:29.516191Z","iopub.status.idle":"2025-09-02T05:26:29.522319Z","shell.execute_reply.started":"2025-09-02T05:26:29.516171Z","shell.execute_reply":"2025-09-02T05:26:29.521719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v17:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 80\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n out += shortcut\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (9 + 4 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 3, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 3, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 256\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:]) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:]) * mask_all.transpose(1, 2)\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA\n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n \n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n\n \n X[BS//8:, :, 14:] = 0.0\n \n \n X, X_imuonly, y = X.float().to(device), X_imuonly.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n \n optimizer.zero_grad()\n \n LEN = 170 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n \n loss.backward()\n \n optimizer.step()\n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n \n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds, val_targets = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_all, val_targets_all = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n \n logits = model(X)\n logits_imuonly = model(X_imuonly)\n \n val_preds.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n\n val_preds_logits.append(logits.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_preds_all.append(logits.argmax(dim=1).cpu().numpy())\n val_preds_all.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_targets.append(y.argmax(dim=1).cpu().numpy())\n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n \n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n val_targets_all.append(y.argmax(dim=1).cpu().numpy())\n \n loss = F.cross_entropy(logits, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_split = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_all)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_all)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ALL : ', round(val_acc, 4), '| SPLIT : ', round(val_acc_split, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n if self.tof_mode == -1:\n for mode in [2, 4, 8, 16, 32]:\n region_size = 64 // mode\n for r in (range(mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{mode}_{i}_region_{r}_std': region_data.std(axis=1),\n f'tof{mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{mode}_{i}_region_{r}_max': region_data.max(axis=1)\n })\n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p = model(x)\n if outputs is None: outputs = p\n else: outputs += p\n outputs /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.523322Z","iopub.execute_input":"2025-09-02T05:26:29.523548Z","iopub.status.idle":"2025-09-02T05:26:29.654493Z","shell.execute_reply.started":"2025-09-02T05:26:29.523532Z","shell.execute_reply":"2025-09-02T05:26:29.653674Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v17_1 = model_zhou_v17('/kaggle/input/cmi3v63/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v17_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_17_1, pred_imuonly_17_1, targets, idlist_17_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_17_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_17_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.6555Z","iopub.execute_input":"2025-09-02T05:26:29.65574Z","iopub.status.idle":"2025-09-02T05:26:29.661495Z","shell.execute_reply.started":"2025-09-02T05:26:29.655713Z","shell.execute_reply":"2025-09-02T05:26:29.660705Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v20:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=1, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n # out = self.act(out)\n\n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n \n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(256, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(256, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256 + 256 + 32 + 64\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 256+128\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.GRU(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n\n self.encoder = ResidualSECNNBlock(self.emb_all, self.dim_lstm, 5)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n\n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n\n # lstm_out = self.encoder(merged).transpose(1,2)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits, logits3\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n \n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n\n \n def train(self, ):\n tof_mode = self.tof_mode\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n\n ratio = np.random.random() *0.85+0.3\n \n #X[:,:3] = X[:,:3] * ratio\n #X[:,7:7+3] = X[:,7:7+3] * ratio\n #X[:,14:14+5] = X[:,14:14+5] * ratio\n\n #X[:,14+10+5:] = X[:,14+10+5:] * ratio\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=True, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n\n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,14:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n X[:BS//8*2,:,7+3:7+7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n X[:BS//8*2,:,7+0:7+3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 14:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n X_accmask[:, :, 7+0:7+3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 14:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n X_rotmask[:, :, 7+3:7+7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n loss = F.cross_entropy(logits_imuonly, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/20):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n\n df[['acc_x', 'acc_y', 'acc_z']] = df[['acc_x', 'acc_y', 'acc_z']] / 5.6\n df[['f1', 'f2', 'f3',]] = df[['f1', 'f2', 'f3',]] / 5.6\n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 3. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 4 - df[pixel_cols].replace(-1, 255.) / 82. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n\n\n # cols_extra = ['acc_x', 'acc_y', 'acc_z', 'f1', 'f2', 'f3'] + [col for col in df.columns if 'thm_' in col ] + [col for col in df.columns if 'tof' in col and '_isna' not in col ]\n \n # group_list = []\n # dict_sid = {}\n # for sid, group in tqdm(df.groupby('sequence_id')):\n # std_all = group[['acc_x', 'acc_y', 'acc_z',]].std(0).mean()\n # dict_sid[sid] = std_all\n\n # df['sid_std'] = df['sequence_id'].map(dict_sid)\n # for col in tqdm(cols_extra):\n # df[col] = df[col].values / (df['sid_std'] + 1e-6)\n # del df['sid_std']\n\n tof_cols = list(df_tof.columns)\n tof_cols_head = tof_cols[:: (self.tof_mode * 2 + 4) ] + tof_cols[1:: (self.tof_mode * 2 + 4) ]\n tof_cols = tof_cols_head + [item for item in tof_cols if item not in tof_cols_head]\n \n return df, feature_names, tof_cols\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.662884Z","iopub.execute_input":"2025-09-02T05:26:29.663066Z","iopub.status.idle":"2025-09-02T05:26:29.800622Z","shell.execute_reply.started":"2025-09-02T05:26:29.663053Z","shell.execute_reply":"2025-09-02T05:26:29.799708Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v20_1 = model_zhou_v20('/kaggle/input/cmi3v75/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v20_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_20_1, pred_imuonly_20_1, targets, idlist_20_1, pred_orientation_20_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_20_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_20_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.801495Z","iopub.execute_input":"2025-09-02T05:26:29.801736Z","iopub.status.idle":"2025-09-02T05:26:29.808183Z","shell.execute_reply.started":"2025-09-02T05:26:29.801718Z","shell.execute_reply":"2025-09-02T05:26:29.807383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v21:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n # out = self.act(out)\n\n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 320, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(320, 512, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 320, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(320, 512, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 128, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(128, 128, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 512 + 512 + 64 + 128\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.GRU(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits, logits3\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n\n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,14:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n X[:BS//8*2,:,7+3:7+7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n X[:BS//8*2,:,7+0:7+3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 14:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n X_accmask[:, :, 7+0:7+3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 14:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n X_rotmask[:, :, 7+3:7+7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n loss = F.cross_entropy(logits_imuonly, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n \n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0)\n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.809342Z","iopub.execute_input":"2025-09-02T05:26:29.809559Z","iopub.status.idle":"2025-09-02T05:26:29.945401Z","shell.execute_reply.started":"2025-09-02T05:26:29.809543Z","shell.execute_reply":"2025-09-02T05:26:29.944576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v21_1 = model_zhou_v21('/kaggle/input/cmi3v78/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v21_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_21_1, pred_imuonly_21_1, targets, idlist_21_1, pred_orientation_21_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_21_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_21_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:29.946314Z","iopub.execute_input":"2025-09-02T05:26:29.946562Z","iopub.status.idle":"2025-09-02T05:26:29.952499Z","shell.execute_reply.started":"2025-09-02T05:26:29.946545Z","shell.execute_reply":"2025-09-02T05:26:29.951835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n \ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v22:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d):\n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n # out = self.act(out)\n\n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.feature_engineering = feature_engineering\n if feature_engineering:\n self.imu_fe = ImuFeatureExtractor(**kwargs)\n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7\n else:\n self.imu_fe = nn.Identity()\n imu_dim = imu_dim_raw \n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7 + 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 320, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(320, 512, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 320, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(320, 512, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 128, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(128, 128, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 512 + 512 + 64 + 128\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.GRU(self.emb_all, self.dim_lstm, num_layers=1, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 256, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(256)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(256, 5)\n self.classifier = nn.Linear(256, n_classes)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n \n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan//2].transpose(1, 2) # (batch, imu_dim, seq_len)\n imu2 = x[:, :, self.fir_nchan//2:self.fir_nchan].transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale3[:,:thm.shape[1],:] + self.bias3[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale4[:,:tof.shape[1],:] + self.bias4[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n\n \n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = torch.cat((x1, x12, thm, tof), 1)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits, logits3\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n\n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {'Seated Lean Non Dom - FACE DOWN': 1,\n 'Lie on Side - Non Dominant': 2,\n 'Seated Straight': 3,\n 'Lie on Back': 4}\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n targetsorientations = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,14:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n X[:BS//8*2,:,7+3:7+7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n X[:BS//8*2,:,7+0:7+3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 14:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 14:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n X_accmask[:, :, 7+0:7+3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 14:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n X_rotmask[:, :, 7+3:7+7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n loss = F.cross_entropy(logits_imuonly, y)\n val_loss += loss.item()\n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n val_loss /= len(val_loader)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n targetsorientations.append(np.concatenate(val_targets_orientation))\n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0),\n 'target_orientation' : np.concatenate(targetsorientations), \n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0), np.concatenate(targetsorientations)\n \n def get_new_features(self, df):\n # 优化后的函数\n def remove_gravity_from_acc(acc_data, rot_data):\n if isinstance(acc_data, pd.DataFrame):\n acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n else:\n acc_values = np.asarray(acc_data)\n \n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = acc_values.shape[0]\n linear_accel = np.zeros_like(acc_values)\n gravity_world = np.array([0, 0, 9.81])\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n \n valid_quats = quat_values[valid_mask]\n valid_quats_normalized = valid_quats / quat_norms[valid_mask, np.newaxis]\n try:\n rotations = R.from_quat(valid_quats_normalized)\n \n gravity_sensor_frame = rotations.apply(gravity_world, inverse=True)\n \n linear_accel[valid_mask] = acc_values[valid_mask] - gravity_sensor_frame\n except:\n linear_accel[valid_mask] = acc_values[valid_mask]\n linear_accel[~valid_mask] = acc_values[~valid_mask]\n \n return linear_accel\n \n \n def calculate_angular_velocity_from_quat(rot_data, time_delta=1/200):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_vel = np.zeros((num_samples, 3))\n \n if num_samples < 2:\n return angular_vel\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q_t = quat_values[:-1][valid_pairs_mask]\n q_t_plus_dt = quat_values[1:][valid_pairs_mask]\n \n q_t_norms = quat_norms[:-1][valid_pairs_mask]\n q_t_plus_dt_norms = quat_norms[1:][valid_pairs_mask]\n \n q_t_norm = q_t / q_t_norms[:, np.newaxis]\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms[:, np.newaxis]\n \n rot_t = R.from_quat(q_t_norm)\n rot_t_plus_dt = R.from_quat(q_t_plus_dt_norm)\n delta_rot = rot_t.inv() * rot_t_plus_dt\n \n angular_vel[:-1][valid_pairs_mask] = delta_rot.as_rotvec() / time_delta\n \n angular_vel[-1, :] = angular_vel[-2, :] if num_samples > 1 else 0\n \n return angular_vel\n \n \n def calculate_angular_distance(rot_data, cumulative=False):\n if isinstance(rot_data, pd.DataFrame):\n quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n else:\n quat_values = np.asarray(rot_data)\n \n num_samples = quat_values.shape[0]\n angular_dist = np.zeros(num_samples)\n \n if num_samples < 2:\n return angular_dist\n \n quat_norms = np.linalg.norm(quat_values, axis=1)\n valid_mask = ~(np.isnan(quat_norms) | np.isclose(quat_norms, 0))\n valid_pairs_mask = valid_mask[:-1] & valid_mask[1:]\n \n if np.any(valid_pairs_mask):\n q1 = quat_values[:-1][valid_pairs_mask]\n q2 = quat_values[1:][valid_pairs_mask]\n \n q1_norms = quat_norms[:-1][valid_pairs_mask]\n q2_norms = quat_norms[1:][valid_pairs_mask]\n \n q1_norm = q1 / q1_norms[:, np.newaxis]\n q2_norm = q2 / q2_norms[:, np.newaxis]\n \n r1 = R.from_quat(q1_norm)\n r2 = R.from_quat(q2_norm)\n relative_rotation = r1.inv() * r2\n \n angular_dist[:-1][valid_pairs_mask] = np.linalg.norm(relative_rotation.as_rotvec(), axis=1)\n \n if num_samples > 1:\n angular_dist[-1] = angular_dist[-2] if cumulative else 0\n \n if cumulative:\n angular_dist = np.cumsum(angular_dist)\n \n return angular_dist\n \n \n \n linear_accel_list = []\n angular_vel_list = []\n angular_distance_list = []\n \n for _, group in (df.groupby('sequence_id')):\n acc_data_group = group[['acc_x', 'acc_y', 'acc_z']].values\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n linear_accel_group = remove_gravity_from_acc(acc_data_group, rot_data_group)\n linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['f1', 'f2', 'f3'], index=group.index))\n\n ot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_vel_group = calculate_angular_velocity_from_quat(rot_data_group)\n angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['f4', 'f5', 'f6'], index=group.index))\n\n rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n angular_dist_group = calculate_angular_distance(rot_data_group)\n angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['f7'], index=group.index))\n \n df_linear_accel = pd.concat(linear_accel_list) \n df_angular_vel = pd.concat(angular_vel_list) \n df_angular_distance = pd.concat(angular_distance_list)\n df = pd.concat([df, df_linear_accel, df_angular_vel, df_angular_distance], axis=1)\n \n feature_names = ['f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7']\n \n \n thm_mean = df[['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']].mean(1)\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = (np.where(df[col].isna(), thm_mean, df[col]) ) / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,14:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-09-02T05:26:29.953657Z","iopub.execute_input":"2025-09-02T05:26:29.953842Z","iopub.status.idle":"2025-09-02T05:26:30.089336Z","shell.execute_reply.started":"2025-09-02T05:26:29.953827Z","shell.execute_reply":"2025-09-02T05:26:30.08876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v22_1 = model_zhou_v22('/kaggle/input/cmi3v79', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v22_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_22_1, pred_imuonly_22_1, targets, idlist_22_1, pred_orientation_22_1, target_orientation_22_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_22_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_22_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.090445Z","iopub.execute_input":"2025-09-02T05:26:30.090718Z","iopub.status.idle":"2025-09-02T05:26:30.096576Z","shell.execute_reply.started":"2025-09-02T05:26:30.090699Z","shell.execute_reply":"2025-09-02T05:26:30.095988Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n\ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v25:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d): \n def get_gravity(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = np.concatenate([g_x, g_y, g_z], -1)\n return gravity\n \n def rotate_quaternion_np(quat: np.ndarray, angle_range: tuple = [(-10, 10), (-10, 10), (-10, 10), ]) -> np.ndarray:\n # 确保输入形状正确\n if quat.ndim != 2 or quat.shape[1] != 4:\n raise ValueError(\"输入四元数必须是形状为[len, 4]的数组\")\n \n seq_len = quat.shape[0]\n \n # 1. 生成绕x、y、z轴的随机旋转角度（度）\n angle_x = np.random.uniform(angle_range[0][0], angle_range[0][1])\n angle_y = np.random.uniform(angle_range[1][0], angle_range[1][1])\n angle_z = np.random.uniform(angle_range[2][0], angle_range[2][1])\n \n # 2. 将角度转换为弧度\n rad_x = math.pi * angle_x / 180.0\n rad_y = math.pi * angle_y / 180.0\n rad_z = math.pi * angle_z / 180.0\n \n # 3. 生成绕各轴旋转的四元数\n qx = np.array([math.sin(rad_x/2), 0, 0, math.cos(rad_x/2)], dtype=np.float32) # x轴旋转\n qy = np.array([0, math.sin(rad_y/2), 0, math.cos(rad_y/2)], dtype=np.float32) # y轴旋转\n qz = np.array([0, 0, math.sin(rad_z/2), math.cos(rad_z/2)], dtype=np.float32) # z轴旋转\n \n # 4. 组合旋转四元数 (q_total = qz * qy * qx，注意乘法顺序)\n q_zy = _quat_mul(qz, qy)\n q_total = _quat_mul(q_zy, qx)\n \n # 5. 将组合旋转应用到每个四元数上\n # 扩展旋转四元数以匹配序列长度\n q_total_expanded = np.tile(q_total, (seq_len, 1)) # [len, 4]\n rotated_quat = _quat_mul(q_total_expanded, quat)\n \n # 6. 归一化确保四元数有效性\n norms = np.linalg.norm(rotated_quat, axis=1, keepdims=True)\n rotated_quat = rotated_quat / np.maximum(norms, 1e-8) # 防止除零\n \n return rotated_quat\n \n def _quat_mul(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:\n # 提取分量（利用广播机制自动适配维度）\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n # 四元数乘法公式（完全向量化）\n x = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2\n y = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2\n z = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2\n w = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2\n \n # 堆叠结果（保持维度）\n return np.stack([x, y, z, w], axis=-1).astype(q1.dtype)\n \n \n def remove_average_rotation_optimized(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 从四元数序列中移除平均旋转，忽略全零四元数，并旋转到指定方向\n Args:\n q: 输入四元数序列，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n eps: 数值稳定性阈值\n Returns:\n q_final: 移除平均旋转并旋转到[0.5,0.5,0.5,0.5]方向后的四元数序列，形状 [bs, len, 4]\n 原始全零的位置仍为零\n \"\"\"\n def is_zero_quaternion(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 判断四元数是否为全零（或接近全零）\n Args:\n q: 四元数，形状 [..., 4]\n eps: 接近零的阈值\n Returns:\n 布尔张量，形状 [...], 指示每个四元数是否为零\n \"\"\"\n return torch.norm(q, dim=-1) < eps\n \n def safe_normalize(q: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:\n \"\"\"\n 安全地规范化四元数，避免除零错误\n Args:\n q: 四元数，形状 [..., 4]\n eps: 最小范数阈值\n Returns:\n 规范化后的四元数\n \"\"\"\n norm = torch.norm(q, dim=-1, keepdim=True)\n # 添加最小范数保护\n norm = torch.clamp(norm, min=eps)\n return q / norm\n \n def quat_multiply(q1: torch.Tensor, q2: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 批量四元数乘法 (q1 * q2)\n Args:\n q1: 形状 [..., 4]，四元数 [x1,y1,z1,w1]\n q2: 形状 [..., 4]，四元数 [x2,y2,z2,w2]\n Returns:\n q: 形状 [..., 4]，乘积 q1*q2 [x,y,z,w]\n \"\"\"\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n x = w1*x2 + x1*w2 + y1*z2 - z1*y2\n y = w1*y2 - x1*z2 + y1*w2 + z1*x2\n z = w1*z2 + x1*y2 - y1*x2 + z1*w2\n w = w1*w2 - x1*x2 - y1*y2 - z1*z2\n \n return torch.stack([x, y, z, w], dim=-1)\n \n def find_first_valid_indices(mask: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 向量化方法查找每个batch中第一个有效四元数的索引\n Args:\n mask: 有效掩码，形状 [bs, len]\n Returns:\n indices: 每个batch的第一个有效索引，形状 [bs]\n \"\"\"\n bs, seq_len = mask.shape\n device = mask.device\n \n # 创建索引矩阵\n indices = torch.arange(seq_len, device=device).expand(bs, seq_len)\n # 将无效位置的索引设置为大数\n indices = torch.where(mask, indices, seq_len)\n # 找到最小索引（第一个有效位置）\n first_valid = torch.min(indices, dim=1)[0]\n # 处理全无效的情况\n all_invalid = first_valid == seq_len\n first_valid[all_invalid] = 0 # 设置为0，后续会处理\n \n return first_valid\n \n def quaternion_average_frechet_optimized(q: torch.Tensor, mask: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算批量四元数的Frechet均值（优化版本）\n Args:\n q: 输入四元数，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n mask: 有效四元数掩码，形状 [bs, len]，1表示有效，0表示无效\n eps: 数值稳定性阈值\n Returns:\n q_avg: 平均四元数，形状 [bs, 4]\n \"\"\"\n bs, len_q, _ = q.shape\n device = q.device\n \n # 步骤1：安全地单位化四元数\n q_normalized = safe_normalize(q, eps=eps)\n \n # 应用掩码：无效四元数置为零\n mask_keepdim = mask.unsqueeze(-1) # [bs, len, 1]\n q_normalized = q_normalized * mask_keepdim\n \n # 步骤2：向量化查找参考四元数\n first_valid_indices = find_first_valid_indices(mask)\n q_ref = q_normalized[torch.arange(bs, device=device), first_valid_indices].unsqueeze(1) # [bs, 1, 4]\n \n # 步骤3：改进的符号统一策略（添加阈值避免在接近正交时翻转）\n dot = (q_normalized * q_ref).sum(dim=-1, keepdim=True) # [bs, len, 1]\n q_unified = torch.where(dot < -0.01, -q_normalized, q_normalized)\n \n # 步骤4：构建协方差矩阵 M [bs, 4, 4]，考虑掩码\n q_reshaped = q_unified.unsqueeze(-1) # [bs, len, 4, 1]\n q_transposed = q_reshaped.transpose(2, 3) # [bs, len, 1, 4]\n outer_products = q_reshaped @ q_transposed # [bs, len, 4, 4]\n \n # 应用掩码并求和\n mask_4d = mask.unsqueeze(-1).unsqueeze(-1) # [bs, len, 1, 1]\n outer_products = outer_products * mask_4d\n M = outer_products.sum(dim=1) # [bs, 4, 4]\n \n # 处理所有四元数都无效的情况\n valid_count = mask.sum(dim=1) # [bs]\n all_invalid = valid_count == 0\n if all_invalid.any():\n # 对全无效的batch，使用单位四元数作为平均\n M[all_invalid] = torch.eye(4, device=device)\n \n # 步骤5：使用更稳定的特征值求解\n # 添加小单位矩阵增强数值稳定性\n M_stable = M + eps * torch.eye(4, device=device).unsqueeze(0)\n w, v = torch.linalg.eigh(M_stable) # w: [bs,4], v: [bs,4,4]\n q_avg = v[:, :, -1] # 最大特征值对应特征向量\n \n # 确保单位化 + 实部(rot_w)为正\n q_avg = safe_normalize(q_avg, eps=eps)\n q_avg = torch.where(q_avg[:, 3:4] < 0, -q_avg, q_avg)\n \n return q_avg\n \n bs, len_q, _ = q.shape\n device = q.device\n \n # 1. 识别全零四元数\n zero_mask = is_zero_quaternion(q, eps) # [bs, len]\n valid_mask = ~zero_mask # [bs, len]，1表示有效四元数\n \n # 2. 计算有效四元数的平均旋转\n q_avg = quaternion_average_frechet_optimized(q, valid_mask, eps) # [bs, 4]\n \n # 3. 创建目标四元数 [0.5, 0.5, 0.5, 0.5] 并规范化\n target_quat = torch.tensor([0, 0, 0, 1], device=device, dtype=q.dtype)\n target_quat = safe_normalize(target_quat)\n \n # 4. 将目标旋转整合到平均旋转中\n # 计算目标旋转与平均旋转的复合旋转\n # q_composite = target_quat * q_avg_conj\n # 这等价于先应用平均旋转的逆，然后应用目标旋转\n \n # 计算平均四元数的共轭（逆）\n q_avg_conj = q_avg.clone()\n q_avg_conj[:, :3] *= -1 # 虚部取反：[x,y,z,w] → [-x,-y,-z,w]\n \n # 将目标旋转与平均旋转的共轭相乘\n q_composite = quat_multiply(\n target_quat.unsqueeze(0).expand(bs, 4), # 扩展目标四元数以匹配批次大小\n q_avg_conj\n ) # [bs, 4]\n \n q_composite = safe_normalize(q_composite) # 确保复合旋转是单位四元数\n q_composite = q_composite.unsqueeze(1) # [bs, 1, 4]，便于广播\n \n # 5. 应用复合旋转：q_final = q_composite * q\n q_final = quat_multiply(q_composite, q) # [bs, len, 4]\n \n # 6. 保持原始全零位置为零\n q_final = q_final * valid_mask.unsqueeze(-1) # [bs, len, 4]\n \n return q_final\n\n \n class MotionFeatureExtractor(nn.Module):\n \"\"\"\n 支持CUDA/CPU输入的运动特征提取模块\n 输入: [bs, len, 7] 张量（[acc_x, acc_y, acc_z, rot_x, rot_y, rot_z, rot_w]）\n 输出: [bs, len, 7] 张量（[线性加速度x/y/z, 角速度x/y/z, 角距离]）\n \"\"\"\n def __init__(self, time_delta=1/200):\n super().__init__()\n self.time_delta = time_delta\n # ！修改1：不提前固定gravity_world的设备，改为使用时动态匹配输入设备\n self.gravity_world_val = torch.tensor([0.0, 0.0, 9.81], dtype=torch.float32)\n \n def quat_to_rot_matrix(self, quat):\n \"\"\"四元数（[x,y,z,w]）转旋转矩阵 [bs, len, 3, 3]\"\"\"\n x, y, z, w = quat[..., 0], quat[..., 1], quat[..., 2], quat[..., 3]\n \n xx = x * x\n yy = y * y\n zz = z * z\n xy = x * y\n xz = x * z\n yz = y * z\n xw = x * w\n yw = y * w\n zw = z * w\n ww = w * w\n \n # 旋转矩阵计算（基于输入quat的设备，自动在CUDA/CPU上运算）\n rot_mat = torch.stack([\n xx + ww - (yy + zz), 2 * (xy - zw), 2 * (xz + yw),\n 2 * (xy + zw), yy + ww - (xx + zz), 2 * (yz - xw),\n 2 * (xz - yw), 2 * (yz + xw), zz + ww - (xx + yy)\n ], dim=-1).view(*quat.shape[:-1], 3, 3)\n \n return rot_mat\n \n def rot_matrix_inverse(self, rot_mat):\n \"\"\"旋转矩阵求逆（逆=转置，设备与输入一致）\"\"\"\n return rot_mat.transpose(-2, -1)\n \n def rot_matrix_multiply(self, rot_mat1, rot_mat2):\n \"\"\"旋转矩阵乘法（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat1, rot_mat2)\n \n def apply_rotation(self, rot_mat, vec):\n \"\"\"旋转矩阵应用于向量（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat, vec.unsqueeze(-1)).squeeze(-1)\n \n def remove_gravity_from_acc(self, acc_data, rot_data):\n \"\"\"从加速度中移除重力（核心：设备动态匹配）\"\"\"\n bs, seq_len, _ = acc_data.shape\n device = acc_data.device # ！修改2：获取输入设备（CUDA/CPU）\n \n valid_quats_normalized = rot_data / torch.norm(rot_data, dim=-1, keepdim=True).clamp(min=1e-8)\n rotations = self.quat_to_rot_matrix(valid_quats_normalized)\n \n # ！修改3：gravity_world动态匹配输入设备，避免CPU/CUDA不兼容\n gravity_world = self.gravity_world_val.to(device).expand(bs, seq_len, 3)\n # 重力向量从世界坐标系转到传感器坐标系\n gravity_sensor_frame = self.apply_rotation(\n self.rot_matrix_inverse(rotations),\n gravity_world\n )\n \n # 线性加速度计算（设备与输入一致）\n linear_accel = acc_data - gravity_sensor_frame\n return linear_accel\n \n def calculate_angular_velocity_from_quat(self, rot_data):\n \"\"\"从四元数计算角速度（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改4：获取输入设备\n angular_vel = torch.zeros(bs, seq_len, 3, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_vel\n \n # 四元数有效性判断（torch.tensor(0.0)改为匹配设备）\n quat_norms = torch.norm(rot_data, dim=-1)\n # ！修改5：用rot_data.new_zeros(1)创建同设备的0张量，替代硬编码的torch.tensor(0.0)\n valid_mask = ~(torch.isnan(quat_norms) | torch.isclose(quat_norms, rot_data.new_zeros(1)))\n valid_pairs_mask = valid_mask[:, :-1] & valid_mask[:, 1:]\n \n if torch.any(valid_pairs_mask):\n # 提取有效四元数对\n q_t = rot_data[:, :-1][valid_pairs_mask]\n q_t_plus_dt = rot_data[:, 1:][valid_pairs_mask]\n \n # 归一化\n q_t_norms = quat_norms[:, :-1][valid_pairs_mask].unsqueeze(-1)\n q_t_plus_dt_norms = quat_norms[:, 1:][valid_pairs_mask].unsqueeze(-1)\n q_t_norm = q_t / q_t_norms.clamp(min=1e-8)\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms.clamp(min=1e-8)\n \n # 旋转矩阵与相对旋转计算\n rot_t = self.quat_to_rot_matrix(q_t_norm)\n rot_t_plus_dt = self.quat_to_rot_matrix(q_t_plus_dt_norm)\n delta_rot = self.rot_matrix_multiply(self.rot_matrix_inverse(rot_t), rot_t_plus_dt)\n \n # 罗德里格斯公式求旋转向量（设备自动匹配）\n trace = delta_rot[..., 0, 0] + delta_rot[..., 1, 1] + delta_rot[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n sin_theta = torch.sin(theta)\n \n # 旋转向量计算（零张量指定设备）\n rot_vec = torch.zeros_like(q_t_norm[..., :3], device=device)\n # ！修改6：同修改5，避免设备不兼容\n mask = ~torch.isclose(sin_theta, rot_data.new_zeros(1))\n \n if torch.any(mask):\n factor = theta / (2 * sin_theta)\n rot_vec[mask] = factor[mask, None] * torch.stack([\n delta_rot[mask, 2, 1] - delta_rot[mask, 1, 2],\n delta_rot[mask, 0, 2] - delta_rot[mask, 2, 0],\n delta_rot[mask, 1, 0] - delta_rot[mask, 0, 1]\n ], dim=1)\n \n # 角速度计算\n angular_vel[:, :-1][valid_pairs_mask] = rot_vec / self.time_delta\n \n # 最后一个时间步复制前一个值\n if seq_len > 1:\n angular_vel[:, -1] = angular_vel[:, -2]\n \n return angular_vel\n \n def calculate_angular_distance(self, rot_data, cumulative=False):\n \"\"\"计算角距离（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改7：获取输入设备\n angular_dist = torch.zeros(bs, seq_len, 1, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_dist\n \n # 四元数归一化\n quat_norms = torch.norm(rot_data, dim=-1)\n q1 = rot_data[:, :-1]\n q2 = rot_data[:, 1:]\n q1_norms = quat_norms[:, :-1].unsqueeze(-1)\n q2_norms = quat_norms[:, 1:].unsqueeze(-1)\n q1_norm = q1 / q1_norms.clamp(min=1e-8)\n q2_norm = q2 / q2_norms.clamp(min=1e-8)\n \n # 相对旋转与角距离计算\n r1 = self.quat_to_rot_matrix(q1_norm)\n r2 = self.quat_to_rot_matrix(q2_norm)\n relative_rotation = self.rot_matrix_multiply(self.rot_matrix_inverse(r1), r2)\n \n trace = relative_rotation[..., 0, 0] + relative_rotation[..., 1, 1] + relative_rotation[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n angular_dist[:, :-1, 0] = theta\n \n # 最后一个时间步处理\n if seq_len > 1:\n angular_dist[:, -1] = angular_dist[:, -2] if cumulative else 0.0\n \n # 累积角距离（设备一致）\n if cumulative:\n angular_dist = torch.cumsum(angular_dist, dim=1)\n \n return angular_dist\n \n def forward(self, x):\n \"\"\"前向传播（自动适配CUDA/CPU输入）\"\"\"\n # 分离加速度（前3维）和四元数（后4维）\n acc_data = x[..., :3]\n rot_data = x[..., 3:]\n \n # 计算三大特征（设备自动匹配输入）\n linear_accel = self.remove_gravity_from_acc(acc_data, rot_data)\n angular_vel = self.calculate_angular_velocity_from_quat(rot_data)\n angular_dist = self.calculate_angular_distance(rot_data)\n \n # 拼接输出（设备与输入一致）\n features = torch.cat([linear_accel, angular_vel, angular_dist], dim=-1)\n return features\n \n\n \n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128,\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n\n out = self.act(out)\n \n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n def get_gravity_torch(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = torch.concat([g_x, g_y, g_z], -1)\n return gravity\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.imu_fe1 = ImuFeatureExtractor(**kwargs)\n self.imu_fe2 = ImuFeatureExtractor(**kwargs)\n self.imu_fe3 = ImuFeatureExtractor(**kwargs)\n \n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7 \n\n self.new_feat_module = MotionFeatureExtractor()\n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n\n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=2, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 512, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(512, 5)\n self.classifier = nn.Linear(512, n_classes * 4)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale5 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias5 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale6 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias6 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan] # (batch, imu_dim, seq_len)\n imu2 = self.new_feat_module(imu)\n imu2 = imu2[:,:,:7]\n\n imu = imu.transpose(1, 2)\n imu2 = imu2.transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe1(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe2(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale5[:,:thm.shape[1],:] + self.bias5[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale6[:,:tof.shape[1],:] + self.bias6[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = (x1 + x12 + thm + tof)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits.reshape(logits.shape[0], 4, 18).mean(1), logits.reshape(logits.shape[0], 4, 18).mean(2)\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n \n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n X[:,3:7] = X[:,3:7]/(np.sqrt((X[:,3:7]**2).sum(-1))+1e-6)[:,None]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {\n 'Lie on Back': 0,\n 'Lie on Side - Non Dominant': 1,\n 'Seated Lean Non Dom - FACE DOWN': 2,\n 'Seated Straight': 3,\n }\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n\n\n df['gesture_int'] = df['gesture_int'] + df['oid'] * 18\n\n \n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(4 * len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n\n # return 1 - (logits.softmax(-1) * y).mean()\n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n targetsorientations = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,7:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 7:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 7:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 7:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.reshape(y.shape[0], 4, 18).sum(1).argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n targetsorientations.append(np.concatenate(val_targets_orientation))\n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0),\n 'target_orientation' : np.concatenate(targetsorientations), \n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0), np.concatenate(targetsorientations)\n \n \n def get_new_features(self, df):\n feature_names = []\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = df[col] / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,7:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.097892Z","iopub.execute_input":"2025-09-02T05:26:30.098088Z","iopub.status.idle":"2025-09-02T05:26:30.259986Z","shell.execute_reply.started":"2025-09-02T05:26:30.098073Z","shell.execute_reply":"2025-09-02T05:26:30.25924Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.260911Z","iopub.execute_input":"2025-09-02T05:26:30.261373Z","iopub.status.idle":"2025-09-02T05:26:30.278063Z","shell.execute_reply.started":"2025-09-02T05:26:30.261354Z","shell.execute_reply":"2025-09-02T05:26:30.277487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n\ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v24:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d): \n def get_gravity(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = np.concatenate([g_x, g_y, g_z], -1)\n return gravity\n \n def rotate_quaternion_np(quat: np.ndarray, angle_range: tuple = [(-10, 10), (-10, 10), (-10, 10), ]) -> np.ndarray:\n # 确保输入形状正确\n if quat.ndim != 2 or quat.shape[1] != 4:\n raise ValueError(\"输入四元数必须是形状为[len, 4]的数组\")\n \n seq_len = quat.shape[0]\n \n # 1. 生成绕x、y、z轴的随机旋转角度（度）\n angle_x = np.random.uniform(angle_range[0][0], angle_range[0][1])\n angle_y = np.random.uniform(angle_range[1][0], angle_range[1][1])\n angle_z = np.random.uniform(angle_range[2][0], angle_range[2][1])\n \n # 2. 将角度转换为弧度\n rad_x = math.pi * angle_x / 180.0\n rad_y = math.pi * angle_y / 180.0\n rad_z = math.pi * angle_z / 180.0\n \n # 3. 生成绕各轴旋转的四元数\n qx = np.array([math.sin(rad_x/2), 0, 0, math.cos(rad_x/2)], dtype=np.float32) # x轴旋转\n qy = np.array([0, math.sin(rad_y/2), 0, math.cos(rad_y/2)], dtype=np.float32) # y轴旋转\n qz = np.array([0, 0, math.sin(rad_z/2), math.cos(rad_z/2)], dtype=np.float32) # z轴旋转\n \n # 4. 组合旋转四元数 (q_total = qz * qy * qx，注意乘法顺序)\n q_zy = _quat_mul(qz, qy)\n q_total = _quat_mul(q_zy, qx)\n \n # 5. 将组合旋转应用到每个四元数上\n # 扩展旋转四元数以匹配序列长度\n q_total_expanded = np.tile(q_total, (seq_len, 1)) # [len, 4]\n rotated_quat = _quat_mul(q_total_expanded, quat)\n \n # 6. 归一化确保四元数有效性\n norms = np.linalg.norm(rotated_quat, axis=1, keepdims=True)\n rotated_quat = rotated_quat / np.maximum(norms, 1e-8) # 防止除零\n \n return rotated_quat\n \n def _quat_mul(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:\n # 提取分量（利用广播机制自动适配维度）\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n # 四元数乘法公式（完全向量化）\n x = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2\n y = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2\n z = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2\n w = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2\n \n # 堆叠结果（保持维度）\n return np.stack([x, y, z, w], axis=-1).astype(q1.dtype)\n \n \n def remove_average_rotation_optimized(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 从四元数序列中移除平均旋转，忽略全零四元数，并旋转到指定方向\n Args:\n q: 输入四元数序列，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n eps: 数值稳定性阈值\n Returns:\n q_final: 移除平均旋转并旋转到[0.5,0.5,0.5,0.5]方向后的四元数序列，形状 [bs, len, 4]\n 原始全零的位置仍为零\n \"\"\"\n def is_zero_quaternion(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 判断四元数是否为全零（或接近全零）\n Args:\n q: 四元数，形状 [..., 4]\n eps: 接近零的阈值\n Returns:\n 布尔张量，形状 [...], 指示每个四元数是否为零\n \"\"\"\n return torch.norm(q, dim=-1) < eps\n \n def safe_normalize(q: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:\n \"\"\"\n 安全地规范化四元数，避免除零错误\n Args:\n q: 四元数，形状 [..., 4]\n eps: 最小范数阈值\n Returns:\n 规范化后的四元数\n \"\"\"\n norm = torch.norm(q, dim=-1, keepdim=True)\n # 添加最小范数保护\n norm = torch.clamp(norm, min=eps)\n return q / norm\n \n def quat_multiply(q1: torch.Tensor, q2: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 批量四元数乘法 (q1 * q2)\n Args:\n q1: 形状 [..., 4]，四元数 [x1,y1,z1,w1]\n q2: 形状 [..., 4]，四元数 [x2,y2,z2,w2]\n Returns:\n q: 形状 [..., 4]，乘积 q1*q2 [x,y,z,w]\n \"\"\"\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n x = w1*x2 + x1*w2 + y1*z2 - z1*y2\n y = w1*y2 - x1*z2 + y1*w2 + z1*x2\n z = w1*z2 + x1*y2 - y1*x2 + z1*w2\n w = w1*w2 - x1*x2 - y1*y2 - z1*z2\n \n return torch.stack([x, y, z, w], dim=-1)\n \n def find_first_valid_indices(mask: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 向量化方法查找每个batch中第一个有效四元数的索引\n Args:\n mask: 有效掩码，形状 [bs, len]\n Returns:\n indices: 每个batch的第一个有效索引，形状 [bs]\n \"\"\"\n bs, seq_len = mask.shape\n device = mask.device\n \n # 创建索引矩阵\n indices = torch.arange(seq_len, device=device).expand(bs, seq_len)\n # 将无效位置的索引设置为大数\n indices = torch.where(mask, indices, seq_len)\n # 找到最小索引（第一个有效位置）\n first_valid = torch.min(indices, dim=1)[0]\n # 处理全无效的情况\n all_invalid = first_valid == seq_len\n first_valid[all_invalid] = 0 # 设置为0，后续会处理\n \n return first_valid\n \n def quaternion_average_frechet_optimized(q: torch.Tensor, mask: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算批量四元数的Frechet均值（优化版本）\n Args:\n q: 输入四元数，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n mask: 有效四元数掩码，形状 [bs, len]，1表示有效，0表示无效\n eps: 数值稳定性阈值\n Returns:\n q_avg: 平均四元数，形状 [bs, 4]\n \"\"\"\n bs, len_q, _ = q.shape\n device = q.device\n \n # 步骤1：安全地单位化四元数\n q_normalized = safe_normalize(q, eps=eps)\n \n # 应用掩码：无效四元数置为零\n mask_keepdim = mask.unsqueeze(-1) # [bs, len, 1]\n q_normalized = q_normalized * mask_keepdim\n \n # 步骤2：向量化查找参考四元数\n first_valid_indices = find_first_valid_indices(mask)\n q_ref = q_normalized[torch.arange(bs, device=device), first_valid_indices].unsqueeze(1) # [bs, 1, 4]\n \n # 步骤3：改进的符号统一策略（添加阈值避免在接近正交时翻转）\n dot = (q_normalized * q_ref).sum(dim=-1, keepdim=True) # [bs, len, 1]\n q_unified = torch.where(dot < -0.01, -q_normalized, q_normalized)\n \n # 步骤4：构建协方差矩阵 M [bs, 4, 4]，考虑掩码\n q_reshaped = q_unified.unsqueeze(-1) # [bs, len, 4, 1]\n q_transposed = q_reshaped.transpose(2, 3) # [bs, len, 1, 4]\n outer_products = q_reshaped @ q_transposed # [bs, len, 4, 4]\n \n # 应用掩码并求和\n mask_4d = mask.unsqueeze(-1).unsqueeze(-1) # [bs, len, 1, 1]\n outer_products = outer_products * mask_4d\n M = outer_products.sum(dim=1) # [bs, 4, 4]\n \n # 处理所有四元数都无效的情况\n valid_count = mask.sum(dim=1) # [bs]\n all_invalid = valid_count == 0\n if all_invalid.any():\n # 对全无效的batch，使用单位四元数作为平均\n M[all_invalid] = torch.eye(4, device=device)\n \n # 步骤5：使用更稳定的特征值求解\n # 添加小单位矩阵增强数值稳定性\n M_stable = M + eps * torch.eye(4, device=device).unsqueeze(0)\n w, v = torch.linalg.eigh(M_stable) # w: [bs,4], v: [bs,4,4]\n q_avg = v[:, :, -1] # 最大特征值对应特征向量\n \n # 确保单位化 + 实部(rot_w)为正\n q_avg = safe_normalize(q_avg, eps=eps)\n q_avg = torch.where(q_avg[:, 3:4] < 0, -q_avg, q_avg)\n \n return q_avg\n \n bs, len_q, _ = q.shape\n device = q.device\n \n # 1. 识别全零四元数\n zero_mask = is_zero_quaternion(q, eps) # [bs, len]\n valid_mask = ~zero_mask # [bs, len]，1表示有效四元数\n \n # 2. 计算有效四元数的平均旋转\n q_avg = quaternion_average_frechet_optimized(q, valid_mask, eps) # [bs, 4]\n \n # 3. 创建目标四元数 [0.5, 0.5, 0.5, 0.5] 并规范化\n target_quat = torch.tensor([0, 0, 0, 1], device=device, dtype=q.dtype)\n target_quat = safe_normalize(target_quat)\n \n # 4. 将目标旋转整合到平均旋转中\n # 计算目标旋转与平均旋转的复合旋转\n # q_composite = target_quat * q_avg_conj\n # 这等价于先应用平均旋转的逆，然后应用目标旋转\n \n # 计算平均四元数的共轭（逆）\n q_avg_conj = q_avg.clone()\n q_avg_conj[:, :3] *= -1 # 虚部取反：[x,y,z,w] → [-x,-y,-z,w]\n \n # 将目标旋转与平均旋转的共轭相乘\n q_composite = quat_multiply(\n target_quat.unsqueeze(0).expand(bs, 4), # 扩展目标四元数以匹配批次大小\n q_avg_conj\n ) # [bs, 4]\n \n q_composite = safe_normalize(q_composite) # 确保复合旋转是单位四元数\n q_composite = q_composite.unsqueeze(1) # [bs, 1, 4]，便于广播\n \n # 5. 应用复合旋转：q_final = q_composite * q\n q_final = quat_multiply(q_composite, q) # [bs, len, 4]\n \n # 6. 保持原始全零位置为零\n q_final = q_final * valid_mask.unsqueeze(-1) # [bs, len, 4]\n \n return q_final\n\n \n class MotionFeatureExtractor(nn.Module):\n \"\"\"\n 支持CUDA/CPU输入的运动特征提取模块\n 输入: [bs, len, 7] 张量（[acc_x, acc_y, acc_z, rot_x, rot_y, rot_z, rot_w]）\n 输出: [bs, len, 7] 张量（[线性加速度x/y/z, 角速度x/y/z, 角距离]）\n \"\"\"\n def __init__(self, time_delta=1/200):\n super().__init__()\n self.time_delta = time_delta\n # ！修改1：不提前固定gravity_world的设备，改为使用时动态匹配输入设备\n self.gravity_world_val = torch.tensor([0.0, 0.0, 9.81], dtype=torch.float32)\n \n def quat_to_rot_matrix(self, quat):\n \"\"\"四元数（[x,y,z,w]）转旋转矩阵 [bs, len, 3, 3]\"\"\"\n x, y, z, w = quat[..., 0], quat[..., 1], quat[..., 2], quat[..., 3]\n \n xx = x * x\n yy = y * y\n zz = z * z\n xy = x * y\n xz = x * z\n yz = y * z\n xw = x * w\n yw = y * w\n zw = z * w\n ww = w * w\n \n # 旋转矩阵计算（基于输入quat的设备，自动在CUDA/CPU上运算）\n rot_mat = torch.stack([\n xx + ww - (yy + zz), 2 * (xy - zw), 2 * (xz + yw),\n 2 * (xy + zw), yy + ww - (xx + zz), 2 * (yz - xw),\n 2 * (xz - yw), 2 * (yz + xw), zz + ww - (xx + yy)\n ], dim=-1).view(*quat.shape[:-1], 3, 3)\n \n return rot_mat\n \n def rot_matrix_inverse(self, rot_mat):\n \"\"\"旋转矩阵求逆（逆=转置，设备与输入一致）\"\"\"\n return rot_mat.transpose(-2, -1)\n \n def rot_matrix_multiply(self, rot_mat1, rot_mat2):\n \"\"\"旋转矩阵乘法（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat1, rot_mat2)\n \n def apply_rotation(self, rot_mat, vec):\n \"\"\"旋转矩阵应用于向量（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat, vec.unsqueeze(-1)).squeeze(-1)\n \n def remove_gravity_from_acc(self, acc_data, rot_data):\n \"\"\"从加速度中移除重力（核心：设备动态匹配）\"\"\"\n bs, seq_len, _ = acc_data.shape\n device = acc_data.device # ！修改2：获取输入设备（CUDA/CPU）\n \n valid_quats_normalized = rot_data / torch.norm(rot_data, dim=-1, keepdim=True).clamp(min=1e-8)\n rotations = self.quat_to_rot_matrix(valid_quats_normalized)\n \n # ！修改3：gravity_world动态匹配输入设备，避免CPU/CUDA不兼容\n gravity_world = self.gravity_world_val.to(device).expand(bs, seq_len, 3)\n # 重力向量从世界坐标系转到传感器坐标系\n gravity_sensor_frame = self.apply_rotation(\n self.rot_matrix_inverse(rotations),\n gravity_world\n )\n \n # 线性加速度计算（设备与输入一致）\n linear_accel = acc_data - gravity_sensor_frame\n return linear_accel\n \n def calculate_angular_velocity_from_quat(self, rot_data):\n \"\"\"从四元数计算角速度（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改4：获取输入设备\n angular_vel = torch.zeros(bs, seq_len, 3, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_vel\n \n # 四元数有效性判断（torch.tensor(0.0)改为匹配设备）\n quat_norms = torch.norm(rot_data, dim=-1)\n # ！修改5：用rot_data.new_zeros(1)创建同设备的0张量，替代硬编码的torch.tensor(0.0)\n valid_mask = ~(torch.isnan(quat_norms) | torch.isclose(quat_norms, rot_data.new_zeros(1)))\n valid_pairs_mask = valid_mask[:, :-1] & valid_mask[:, 1:]\n \n if torch.any(valid_pairs_mask):\n # 提取有效四元数对\n q_t = rot_data[:, :-1][valid_pairs_mask]\n q_t_plus_dt = rot_data[:, 1:][valid_pairs_mask]\n \n # 归一化\n q_t_norms = quat_norms[:, :-1][valid_pairs_mask].unsqueeze(-1)\n q_t_plus_dt_norms = quat_norms[:, 1:][valid_pairs_mask].unsqueeze(-1)\n q_t_norm = q_t / q_t_norms.clamp(min=1e-8)\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms.clamp(min=1e-8)\n \n # 旋转矩阵与相对旋转计算\n rot_t = self.quat_to_rot_matrix(q_t_norm)\n rot_t_plus_dt = self.quat_to_rot_matrix(q_t_plus_dt_norm)\n delta_rot = self.rot_matrix_multiply(self.rot_matrix_inverse(rot_t), rot_t_plus_dt)\n \n # 罗德里格斯公式求旋转向量（设备自动匹配）\n trace = delta_rot[..., 0, 0] + delta_rot[..., 1, 1] + delta_rot[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n sin_theta = torch.sin(theta)\n \n # 旋转向量计算（零张量指定设备）\n rot_vec = torch.zeros_like(q_t_norm[..., :3], device=device)\n # ！修改6：同修改5，避免设备不兼容\n mask = ~torch.isclose(sin_theta, rot_data.new_zeros(1))\n \n if torch.any(mask):\n factor = theta / (2 * sin_theta)\n rot_vec[mask] = factor[mask, None] * torch.stack([\n delta_rot[mask, 2, 1] - delta_rot[mask, 1, 2],\n delta_rot[mask, 0, 2] - delta_rot[mask, 2, 0],\n delta_rot[mask, 1, 0] - delta_rot[mask, 0, 1]\n ], dim=1)\n \n # 角速度计算\n angular_vel[:, :-1][valid_pairs_mask] = rot_vec / self.time_delta\n \n # 最后一个时间步复制前一个值\n if seq_len > 1:\n angular_vel[:, -1] = angular_vel[:, -2]\n \n return angular_vel\n \n def calculate_angular_distance(self, rot_data, cumulative=False):\n \"\"\"计算角距离（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改7：获取输入设备\n angular_dist = torch.zeros(bs, seq_len, 1, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_dist\n \n # 四元数归一化\n quat_norms = torch.norm(rot_data, dim=-1)\n q1 = rot_data[:, :-1]\n q2 = rot_data[:, 1:]\n q1_norms = quat_norms[:, :-1].unsqueeze(-1)\n q2_norms = quat_norms[:, 1:].unsqueeze(-1)\n q1_norm = q1 / q1_norms.clamp(min=1e-8)\n q2_norm = q2 / q2_norms.clamp(min=1e-8)\n \n # 相对旋转与角距离计算\n r1 = self.quat_to_rot_matrix(q1_norm)\n r2 = self.quat_to_rot_matrix(q2_norm)\n relative_rotation = self.rot_matrix_multiply(self.rot_matrix_inverse(r1), r2)\n \n trace = relative_rotation[..., 0, 0] + relative_rotation[..., 1, 1] + relative_rotation[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n angular_dist[:, :-1, 0] = theta\n \n # 最后一个时间步处理\n if seq_len > 1:\n angular_dist[:, -1] = angular_dist[:, -2] if cumulative else 0.0\n \n # 累积角距离（设备一致）\n if cumulative:\n angular_dist = torch.cumsum(angular_dist, dim=1)\n \n return angular_dist\n \n def forward(self, x):\n \"\"\"前向传播（自动适配CUDA/CPU输入）\"\"\"\n # 分离加速度（前3维）和四元数（后4维）\n acc_data = x[..., :3]\n rot_data = x[..., 3:]\n \n # 计算三大特征（设备自动匹配输入）\n linear_accel = self.remove_gravity_from_acc(acc_data, rot_data)\n angular_vel = self.calculate_angular_velocity_from_quat(rot_data)\n angular_dist = self.calculate_angular_distance(rot_data)\n \n # 拼接输出（设备与输入一致）\n features = torch.cat([linear_accel, angular_vel, angular_dist], dim=-1)\n return features\n \n\n \n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128,\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n\n out = self.act(out)\n \n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n def get_gravity_torch(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = torch.concat([g_x, g_y, g_z], -1)\n return gravity\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.imu_fe1 = ImuFeatureExtractor(**kwargs)\n self.imu_fe2 = ImuFeatureExtractor(**kwargs)\n self.imu_fe3 = ImuFeatureExtractor(**kwargs)\n \n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7 \n\n self.new_feat_module = MotionFeatureExtractor()\n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n\n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=2, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 512, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(512, 5)\n self.classifier = nn.Linear(512, n_classes * 4)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale5 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias5 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale6 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias6 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan] # (batch, imu_dim, seq_len)\n imu2 = self.new_feat_module(imu)\n imu2 = imu2[:,:,:7]\n\n imu = imu.transpose(1, 2)\n imu2 = imu2.transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe1(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe2(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale5[:,:thm.shape[1],:] + self.bias5[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale6[:,:tof.shape[1],:] + self.bias6[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = (x1 + x12 + thm + tof)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits.reshape(logits.shape[0], 4, 18).mean(1), logits.reshape(logits.shape[0], 4, 18).mean(2)\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n \n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n X[:,3:7] = X[:,3:7]/(np.sqrt((X[:,3:7]**2).sum(-1))+1e-6)[:,None]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {\n 'Lie on Back': 0,\n 'Lie on Side - Non Dominant': 1,\n 'Seated Lean Non Dom - FACE DOWN': 2,\n 'Seated Straight': 3,\n }\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n\n\n df['gesture_int'] = df['gesture_int'] + df['oid'] * 18\n\n \n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(4 * len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n\n # return 1 - (logits.softmax(-1) * y).mean()\n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n targetsorientations = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,7:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 7:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 7:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 7:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.reshape(y.shape[0], 4, 18).sum(1).argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n targetsorientations.append(np.concatenate(val_targets_orientation))\n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0),\n 'target_orientation' : np.concatenate(targetsorientations), \n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0), np.concatenate(targetsorientations)\n \n \n def get_new_features(self, df):\n feature_names = []\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = df[col] / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,7:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, (outputs.cpu().numpy(), outputs2.cpu().numpy())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.279156Z","iopub.execute_input":"2025-09-02T05:26:30.27935Z","iopub.status.idle":"2025-09-02T05:26:30.437585Z","shell.execute_reply.started":"2025-09-02T05:26:30.279335Z","shell.execute_reply":"2025-09-02T05:26:30.43685Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v24_1 = model_zhou_v24('/kaggle/input/cmi3v86/42', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v24_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_24_1, pred_imuonly_24_1, targets, idlist_24_1, pred_orientation_24_1, target_orientation_24_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_24_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_24_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.438478Z","iopub.execute_input":"2025-09-02T05:26:30.438695Z","iopub.status.idle":"2025-09-02T05:26:30.444595Z","shell.execute_reply.started":"2025-09-02T05:26:30.438678Z","shell.execute_reply":"2025-09-02T05:26:30.44393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n\ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v23:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d): \n def get_gravity(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = np.concatenate([g_x, g_y, g_z], -1)\n return gravity\n \n def rotate_quaternion_np(quat: np.ndarray, angle_range: tuple = [(-10, 10), (-10, 10), (-10, 10), ]) -> np.ndarray:\n # 确保输入形状正确\n if quat.ndim != 2 or quat.shape[1] != 4:\n raise ValueError(\"输入四元数必须是形状为[len, 4]的数组\")\n \n seq_len = quat.shape[0]\n \n # 1. 生成绕x、y、z轴的随机旋转角度（度）\n angle_x = np.random.uniform(angle_range[0][0], angle_range[0][1])\n angle_y = np.random.uniform(angle_range[1][0], angle_range[1][1])\n angle_z = np.random.uniform(angle_range[2][0], angle_range[2][1])\n \n # 2. 将角度转换为弧度\n rad_x = math.pi * angle_x / 180.0\n rad_y = math.pi * angle_y / 180.0\n rad_z = math.pi * angle_z / 180.0\n \n # 3. 生成绕各轴旋转的四元数\n qx = np.array([math.sin(rad_x/2), 0, 0, math.cos(rad_x/2)], dtype=np.float32) # x轴旋转\n qy = np.array([0, math.sin(rad_y/2), 0, math.cos(rad_y/2)], dtype=np.float32) # y轴旋转\n qz = np.array([0, 0, math.sin(rad_z/2), math.cos(rad_z/2)], dtype=np.float32) # z轴旋转\n \n # 4. 组合旋转四元数 (q_total = qz * qy * qx，注意乘法顺序)\n q_zy = _quat_mul(qz, qy)\n q_total = _quat_mul(q_zy, qx)\n \n # 5. 将组合旋转应用到每个四元数上\n # 扩展旋转四元数以匹配序列长度\n q_total_expanded = np.tile(q_total, (seq_len, 1)) # [len, 4]\n rotated_quat = _quat_mul(q_total_expanded, quat)\n \n # 6. 归一化确保四元数有效性\n norms = np.linalg.norm(rotated_quat, axis=1, keepdims=True)\n rotated_quat = rotated_quat / np.maximum(norms, 1e-8) # 防止除零\n \n return rotated_quat\n \n def _quat_mul(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:\n # 提取分量（利用广播机制自动适配维度）\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n # 四元数乘法公式（完全向量化）\n x = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2\n y = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2\n z = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2\n w = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2\n \n # 堆叠结果（保持维度）\n return np.stack([x, y, z, w], axis=-1).astype(q1.dtype)\n \n \n def remove_average_rotation_optimized(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 从四元数序列中移除平均旋转，忽略全零四元数，并旋转到指定方向\n Args:\n q: 输入四元数序列，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n eps: 数值稳定性阈值\n Returns:\n q_final: 移除平均旋转并旋转到[0.5,0.5,0.5,0.5]方向后的四元数序列，形状 [bs, len, 4]\n 原始全零的位置仍为零\n \"\"\"\n def is_zero_quaternion(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 判断四元数是否为全零（或接近全零）\n Args:\n q: 四元数，形状 [..., 4]\n eps: 接近零的阈值\n Returns:\n 布尔张量，形状 [...], 指示每个四元数是否为零\n \"\"\"\n return torch.norm(q, dim=-1) < eps\n \n def safe_normalize(q: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:\n \"\"\"\n 安全地规范化四元数，避免除零错误\n Args:\n q: 四元数，形状 [..., 4]\n eps: 最小范数阈值\n Returns:\n 规范化后的四元数\n \"\"\"\n norm = torch.norm(q, dim=-1, keepdim=True)\n # 添加最小范数保护\n norm = torch.clamp(norm, min=eps)\n return q / norm\n \n def quat_multiply(q1: torch.Tensor, q2: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 批量四元数乘法 (q1 * q2)\n Args:\n q1: 形状 [..., 4]，四元数 [x1,y1,z1,w1]\n q2: 形状 [..., 4]，四元数 [x2,y2,z2,w2]\n Returns:\n q: 形状 [..., 4]，乘积 q1*q2 [x,y,z,w]\n \"\"\"\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n x = w1*x2 + x1*w2 + y1*z2 - z1*y2\n y = w1*y2 - x1*z2 + y1*w2 + z1*x2\n z = w1*z2 + x1*y2 - y1*x2 + z1*w2\n w = w1*w2 - x1*x2 - y1*y2 - z1*z2\n \n return torch.stack([x, y, z, w], dim=-1)\n \n def find_first_valid_indices(mask: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 向量化方法查找每个batch中第一个有效四元数的索引\n Args:\n mask: 有效掩码，形状 [bs, len]\n Returns:\n indices: 每个batch的第一个有效索引，形状 [bs]\n \"\"\"\n bs, seq_len = mask.shape\n device = mask.device\n \n # 创建索引矩阵\n indices = torch.arange(seq_len, device=device).expand(bs, seq_len)\n # 将无效位置的索引设置为大数\n indices = torch.where(mask, indices, seq_len)\n # 找到最小索引（第一个有效位置）\n first_valid = torch.min(indices, dim=1)[0]\n # 处理全无效的情况\n all_invalid = first_valid == seq_len\n first_valid[all_invalid] = 0 # 设置为0，后续会处理\n \n return first_valid\n \n def quaternion_average_frechet_optimized(q: torch.Tensor, mask: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算批量四元数的Frechet均值（优化版本）\n Args:\n q: 输入四元数，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n mask: 有效四元数掩码，形状 [bs, len]，1表示有效，0表示无效\n eps: 数值稳定性阈值\n Returns:\n q_avg: 平均四元数，形状 [bs, 4]\n \"\"\"\n bs, len_q, _ = q.shape\n device = q.device\n \n # 步骤1：安全地单位化四元数\n q_normalized = safe_normalize(q, eps=eps)\n \n # 应用掩码：无效四元数置为零\n mask_keepdim = mask.unsqueeze(-1) # [bs, len, 1]\n q_normalized = q_normalized * mask_keepdim\n \n # 步骤2：向量化查找参考四元数\n first_valid_indices = find_first_valid_indices(mask)\n q_ref = q_normalized[torch.arange(bs, device=device), first_valid_indices].unsqueeze(1) # [bs, 1, 4]\n \n # 步骤3：改进的符号统一策略（添加阈值避免在接近正交时翻转）\n dot = (q_normalized * q_ref).sum(dim=-1, keepdim=True) # [bs, len, 1]\n q_unified = torch.where(dot < -0.01, -q_normalized, q_normalized)\n \n # 步骤4：构建协方差矩阵 M [bs, 4, 4]，考虑掩码\n q_reshaped = q_unified.unsqueeze(-1) # [bs, len, 4, 1]\n q_transposed = q_reshaped.transpose(2, 3) # [bs, len, 1, 4]\n outer_products = q_reshaped @ q_transposed # [bs, len, 4, 4]\n \n # 应用掩码并求和\n mask_4d = mask.unsqueeze(-1).unsqueeze(-1) # [bs, len, 1, 1]\n outer_products = outer_products * mask_4d\n M = outer_products.sum(dim=1) # [bs, 4, 4]\n \n # 处理所有四元数都无效的情况\n valid_count = mask.sum(dim=1) # [bs]\n all_invalid = valid_count == 0\n if all_invalid.any():\n # 对全无效的batch，使用单位四元数作为平均\n M[all_invalid] = torch.eye(4, device=device)\n \n # 步骤5：使用更稳定的特征值求解\n # 添加小单位矩阵增强数值稳定性\n M_stable = M + eps * torch.eye(4, device=device).unsqueeze(0)\n w, v = torch.linalg.eigh(M_stable) # w: [bs,4], v: [bs,4,4]\n q_avg = v[:, :, -1] # 最大特征值对应特征向量\n \n # 确保单位化 + 实部(rot_w)为正\n q_avg = safe_normalize(q_avg, eps=eps)\n q_avg = torch.where(q_avg[:, 3:4] < 0, -q_avg, q_avg)\n \n return q_avg\n \n bs, len_q, _ = q.shape\n device = q.device\n \n # 1. 识别全零四元数\n zero_mask = is_zero_quaternion(q, eps) # [bs, len]\n valid_mask = ~zero_mask # [bs, len]，1表示有效四元数\n \n # 2. 计算有效四元数的平均旋转\n q_avg = quaternion_average_frechet_optimized(q, valid_mask, eps) # [bs, 4]\n \n # 3. 创建目标四元数 [0.5, 0.5, 0.5, 0.5] 并规范化\n target_quat = torch.tensor([0, 0, 0, 1], device=device, dtype=q.dtype)\n target_quat = safe_normalize(target_quat)\n \n # 4. 将目标旋转整合到平均旋转中\n # 计算目标旋转与平均旋转的复合旋转\n # q_composite = target_quat * q_avg_conj\n # 这等价于先应用平均旋转的逆，然后应用目标旋转\n \n # 计算平均四元数的共轭（逆）\n q_avg_conj = q_avg.clone()\n q_avg_conj[:, :3] *= -1 # 虚部取反：[x,y,z,w] → [-x,-y,-z,w]\n \n # 将目标旋转与平均旋转的共轭相乘\n q_composite = quat_multiply(\n target_quat.unsqueeze(0).expand(bs, 4), # 扩展目标四元数以匹配批次大小\n q_avg_conj\n ) # [bs, 4]\n \n q_composite = safe_normalize(q_composite) # 确保复合旋转是单位四元数\n q_composite = q_composite.unsqueeze(1) # [bs, 1, 4]，便于广播\n \n # 5. 应用复合旋转：q_final = q_composite * q\n q_final = quat_multiply(q_composite, q) # [bs, len, 4]\n \n # 6. 保持原始全零位置为零\n q_final = q_final * valid_mask.unsqueeze(-1) # [bs, len, 4]\n \n return q_final\n\n \n class MotionFeatureExtractor(nn.Module):\n \"\"\"\n 支持CUDA/CPU输入的运动特征提取模块\n 输入: [bs, len, 7] 张量（[acc_x, acc_y, acc_z, rot_x, rot_y, rot_z, rot_w]）\n 输出: [bs, len, 7] 张量（[线性加速度x/y/z, 角速度x/y/z, 角距离]）\n \"\"\"\n def __init__(self, time_delta=1/200):\n super().__init__()\n self.time_delta = time_delta\n # ！修改1：不提前固定gravity_world的设备，改为使用时动态匹配输入设备\n self.gravity_world_val = torch.tensor([0.0, 0.0, 9.81], dtype=torch.float32)\n \n def quat_to_rot_matrix(self, quat):\n \"\"\"四元数（[x,y,z,w]）转旋转矩阵 [bs, len, 3, 3]\"\"\"\n x, y, z, w = quat[..., 0], quat[..., 1], quat[..., 2], quat[..., 3]\n \n xx = x * x\n yy = y * y\n zz = z * z\n xy = x * y\n xz = x * z\n yz = y * z\n xw = x * w\n yw = y * w\n zw = z * w\n ww = w * w\n \n # 旋转矩阵计算（基于输入quat的设备，自动在CUDA/CPU上运算）\n rot_mat = torch.stack([\n xx + ww - (yy + zz), 2 * (xy - zw), 2 * (xz + yw),\n 2 * (xy + zw), yy + ww - (xx + zz), 2 * (yz - xw),\n 2 * (xz - yw), 2 * (yz + xw), zz + ww - (xx + yy)\n ], dim=-1).view(*quat.shape[:-1], 3, 3)\n \n return rot_mat\n \n def rot_matrix_inverse(self, rot_mat):\n \"\"\"旋转矩阵求逆（逆=转置，设备与输入一致）\"\"\"\n return rot_mat.transpose(-2, -1)\n \n def rot_matrix_multiply(self, rot_mat1, rot_mat2):\n \"\"\"旋转矩阵乘法（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat1, rot_mat2)\n \n def apply_rotation(self, rot_mat, vec):\n \"\"\"旋转矩阵应用于向量（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat, vec.unsqueeze(-1)).squeeze(-1)\n \n def remove_gravity_from_acc(self, acc_data, rot_data):\n \"\"\"从加速度中移除重力（核心：设备动态匹配）\"\"\"\n bs, seq_len, _ = acc_data.shape\n device = acc_data.device # ！修改2：获取输入设备（CUDA/CPU）\n \n valid_quats_normalized = rot_data / torch.norm(rot_data, dim=-1, keepdim=True).clamp(min=1e-8)\n rotations = self.quat_to_rot_matrix(valid_quats_normalized)\n \n # ！修改3：gravity_world动态匹配输入设备，避免CPU/CUDA不兼容\n gravity_world = self.gravity_world_val.to(device).expand(bs, seq_len, 3)\n # 重力向量从世界坐标系转到传感器坐标系\n gravity_sensor_frame = self.apply_rotation(\n self.rot_matrix_inverse(rotations),\n gravity_world\n )\n \n # 线性加速度计算（设备与输入一致）\n linear_accel = acc_data - gravity_sensor_frame\n return linear_accel\n \n def calculate_angular_velocity_from_quat(self, rot_data):\n \"\"\"从四元数计算角速度（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改4：获取输入设备\n angular_vel = torch.zeros(bs, seq_len, 3, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_vel\n \n # 四元数有效性判断（torch.tensor(0.0)改为匹配设备）\n quat_norms = torch.norm(rot_data, dim=-1)\n # ！修改5：用rot_data.new_zeros(1)创建同设备的0张量，替代硬编码的torch.tensor(0.0)\n valid_mask = ~(torch.isnan(quat_norms) | torch.isclose(quat_norms, rot_data.new_zeros(1)))\n valid_pairs_mask = valid_mask[:, :-1] & valid_mask[:, 1:]\n \n if torch.any(valid_pairs_mask):\n # 提取有效四元数对\n q_t = rot_data[:, :-1][valid_pairs_mask]\n q_t_plus_dt = rot_data[:, 1:][valid_pairs_mask]\n \n # 归一化\n q_t_norms = quat_norms[:, :-1][valid_pairs_mask].unsqueeze(-1)\n q_t_plus_dt_norms = quat_norms[:, 1:][valid_pairs_mask].unsqueeze(-1)\n q_t_norm = q_t / q_t_norms.clamp(min=1e-8)\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms.clamp(min=1e-8)\n \n # 旋转矩阵与相对旋转计算\n rot_t = self.quat_to_rot_matrix(q_t_norm)\n rot_t_plus_dt = self.quat_to_rot_matrix(q_t_plus_dt_norm)\n delta_rot = self.rot_matrix_multiply(self.rot_matrix_inverse(rot_t), rot_t_plus_dt)\n \n # 罗德里格斯公式求旋转向量（设备自动匹配）\n trace = delta_rot[..., 0, 0] + delta_rot[..., 1, 1] + delta_rot[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n sin_theta = torch.sin(theta)\n \n # 旋转向量计算（零张量指定设备）\n rot_vec = torch.zeros_like(q_t_norm[..., :3], device=device)\n # ！修改6：同修改5，避免设备不兼容\n mask = ~torch.isclose(sin_theta, rot_data.new_zeros(1))\n \n if torch.any(mask):\n factor = theta / (2 * sin_theta)\n rot_vec[mask] = factor[mask, None] * torch.stack([\n delta_rot[mask, 2, 1] - delta_rot[mask, 1, 2],\n delta_rot[mask, 0, 2] - delta_rot[mask, 2, 0],\n delta_rot[mask, 1, 0] - delta_rot[mask, 0, 1]\n ], dim=1)\n \n # 角速度计算\n angular_vel[:, :-1][valid_pairs_mask] = rot_vec / self.time_delta\n \n # 最后一个时间步复制前一个值\n if seq_len > 1:\n angular_vel[:, -1] = angular_vel[:, -2]\n \n return angular_vel\n \n def calculate_angular_distance(self, rot_data, cumulative=False):\n \"\"\"计算角距离（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改7：获取输入设备\n angular_dist = torch.zeros(bs, seq_len, 1, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_dist\n \n # 四元数归一化\n quat_norms = torch.norm(rot_data, dim=-1)\n q1 = rot_data[:, :-1]\n q2 = rot_data[:, 1:]\n q1_norms = quat_norms[:, :-1].unsqueeze(-1)\n q2_norms = quat_norms[:, 1:].unsqueeze(-1)\n q1_norm = q1 / q1_norms.clamp(min=1e-8)\n q2_norm = q2 / q2_norms.clamp(min=1e-8)\n \n # 相对旋转与角距离计算\n r1 = self.quat_to_rot_matrix(q1_norm)\n r2 = self.quat_to_rot_matrix(q2_norm)\n relative_rotation = self.rot_matrix_multiply(self.rot_matrix_inverse(r1), r2)\n \n trace = relative_rotation[..., 0, 0] + relative_rotation[..., 1, 1] + relative_rotation[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n angular_dist[:, :-1, 0] = theta\n \n # 最后一个时间步处理\n if seq_len > 1:\n angular_dist[:, -1] = angular_dist[:, -2] if cumulative else 0.0\n \n # 累积角距离（设备一致）\n if cumulative:\n angular_dist = torch.cumsum(angular_dist, dim=1)\n \n return angular_dist\n \n def forward(self, x):\n \"\"\"前向传播（自动适配CUDA/CPU输入）\"\"\"\n # 分离加速度（前3维）和四元数（后4维）\n acc_data = x[..., :3]\n rot_data = x[..., 3:]\n \n # 计算三大特征（设备自动匹配输入）\n linear_accel = self.remove_gravity_from_acc(acc_data, rot_data)\n angular_vel = self.calculate_angular_velocity_from_quat(rot_data)\n angular_dist = self.calculate_angular_distance(rot_data)\n \n # 拼接输出（设备与输入一致）\n features = torch.cat([linear_accel, angular_vel, angular_dist], dim=-1)\n return features\n \n\n \n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128,\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n\n out = self.act(out)\n \n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n def get_gravity_torch(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = torch.concat([g_x, g_y, g_z], -1)\n return gravity\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.imu_fe1 = ImuFeatureExtractor(**kwargs)\n self.imu_fe2 = ImuFeatureExtractor(**kwargs)\n self.imu_fe3 = ImuFeatureExtractor(**kwargs)\n \n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7 \n\n self.new_feat_module = MotionFeatureExtractor()\n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n\n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=2, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 512, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(512, 5)\n self.classifier = nn.Linear(512, n_classes * 4)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale5 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias5 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale6 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias6 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan] # (batch, imu_dim, seq_len)\n imu2 = self.new_feat_module(imu)\n imu2 = imu2[:,:,:7]\n\n imu = imu.transpose(1, 2)\n imu2 = imu2.transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe1(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe2(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale5[:,:thm.shape[1],:] + self.bias5[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale6[:,:tof.shape[1],:] + self.bias6[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = (x1 + x12 + thm + tof)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits.reshape(logits.shape[0], 4, 18).mean(1), logits.reshape(logits.shape[0], 4, 18).mean(2)\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n \n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n X[:,3:7] = X[:,3:7]/(np.sqrt((X[:,3:7]**2).sum(-1))+1e-6)[:,None]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {\n 'Lie on Back': 0,\n 'Lie on Side - Non Dominant': 1,\n 'Seated Lean Non Dom - FACE DOWN': 2,\n 'Seated Straight': 3,\n }\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n\n\n df['gesture_int'] = df['gesture_int'] + df['oid'] * 18\n\n \n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(4 * len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n\n # return 1 - (logits.softmax(-1) * y).mean()\n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n targetsorientations = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,7:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 7:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 7:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 7:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, _ = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, logits_orientation = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.reshape(y.shape[0], 4, 18).sum(1).argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n \n train_loss = np.mean(train_loss)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_imuonly # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n targetsorientations.append(np.concatenate(val_targets_orientation))\n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0),\n 'target_orientation' : np.concatenate(targetsorientations), \n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0), np.concatenate(targetsorientations)\n \n def get_new_features(self, df):\n feature_names = []\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = df[col] / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,7:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, outputs.cpu().numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.445884Z","iopub.execute_input":"2025-09-02T05:26:30.446076Z","iopub.status.idle":"2025-09-02T05:26:30.602211Z","shell.execute_reply.started":"2025-09-02T05:26:30.446061Z","shell.execute_reply":"2025-09-02T05:26:30.601662Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:26:30.602816Z","iopub.execute_input":"2025-09-02T05:26:30.603003Z","iopub.status.idle":"2025-09-02T05:26:30.608893Z","shell.execute_reply.started":"2025-09-02T05:26:30.602988Z","shell.execute_reply":"2025-09-02T05:26:30.608135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, json, joblib, numpy as np, pandas as pd\nimport random, math\nfrom pathlib import Path\nimport warnings \nwarnings.filterwarnings(\"ignore\")\n\nimport sys\nsys.path.append('/root/autodl-tmp/')\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import CosineAnnealingWarmRestarts\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.utils.class_weight import compute_class_weight\nfrom sklearn.model_selection import StratifiedKFold\nfrom timm.scheduler import CosineLRScheduler\nfrom scipy.signal import firwin\nimport polars as pl\nimport numpy as np\nimport random\n\n\nimport numpy as np\nimport pandas as pd\nfrom scipy.spatial.transform import Rotation as R\nfrom tqdm.auto import tqdm \n\n\nclass SignalTransform:\n def __init__(self, always_apply: bool = False, p: float = 0.5):\n self.always_apply = always_apply\n self.p = p\n\n def __call__(self, y: np.ndarray):\n if self.always_apply:\n return self.apply(y)\n else:\n if np.random.rand() < self.p:\n return self.apply(y)\n else:\n return y\n\n def apply(self, y: np.ndarray):\n raise NotImplementedError\n\n\nclass Compose:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n for trns in self.transforms:\n y = trns(y)\n return y\n\n\nclass OneOf:\n def __init__(self, transforms: list):\n self.transforms = transforms\n\n def __call__(self, y: np.ndarray):\n n_trns = len(self.transforms)\n trns_idx = np.random.choice(n_trns)\n trns = self.transforms[trns_idx]\n return trns(y)\n\nclass TimeStretch(SignalTransform):\n def __init__(self, max_rate=1.5, min_rate=0.5, always_apply=False, p=0.5):\n super().__init__(always_apply, p)\n self.max_rate = max_rate\n self.min_rate = min_rate\n self.always_apply = always_apply\n self.p = p\n\n def apply(self, x: np.ndarray):\n \"\"\"\n Stretch a 1D or 2D array in time using linear interpolation.\n After stretching, pad or crop at the beginning to match the original length.\n Padding value is 0.\n - x: np.ndarray of shape (L,) or (L, N)\n \"\"\"\n rate = np.random.uniform(self.min_rate, self.max_rate)\n L = x.shape[0]\n L_new = int(L / rate)\n orig_idx = np.linspace(0, L - 1, num=L)\n new_idx = np.linspace(0, L - 1, num=L_new)\n\n if x.ndim == 1:\n stretched = np.interp(new_idx, orig_idx, x)\n # Pad or crop at the beginning\n if L_new < L:\n # Pad at the beginning\n padded = np.zeros(L, dtype=stretched.dtype)\n padded[-L_new:] = stretched\n return padded\n elif L_new > L:\n # Crop at the beginning\n return stretched[-L:]\n else:\n return stretched\n elif x.ndim == 2:\n stretched = np.stack([\n np.interp(new_idx, orig_idx, x[:, i]) for i in range(x.shape[1])\n ], axis=1)\n if L_new < L:\n padded = np.zeros((L, x.shape[1]), dtype=stretched.dtype)\n padded[-L_new:, :] = stretched\n return padded\n elif L_new > L:\n return stretched[-L:, :]\n else:\n return stretched\n else:\n raise ValueError(\"Only 1D or 2D arrays are supported.\")\n\n\nclass TimeShift(SignalTransform):\n def __init__(self, always_apply=False, p=0.5, max_shift_pct=0.25, padding_mode=\"replace\"):\n super().__init__(always_apply, p)\n \n assert 0 <= max_shift_pct <= 1.0, \"`max_shift_pct` must be between 0 and 1\"\n assert padding_mode in [\"replace\", \"zero\"], \"`padding_mode` must be either 'replace' or 'zero'\"\n \n self.max_shift_pct = max_shift_pct\n self.padding_mode = padding_mode\n\n def apply(self, x: np.ndarray, **params):\n assert x.ndim == 2, \"`x` must be a 2D array with shape (L, N)\"\n \n L = x.shape[0]\n max_shift = int(L * self.max_shift_pct)\n shift = np.random.randint(-max_shift, max_shift + 1)\n\n # Roll along time axis (axis=0)\n augmented = np.roll(x, shift, axis=0)\n\n if self.padding_mode == \"zero\":\n if shift > 0:\n augmented[:shift, :] = 0\n elif shift < 0:\n augmented[shift:, :] = 0\n\n return augmented\n\ntransforms_custom = Compose([\n TimeShift(p=0.25, padding_mode=\"replace\", max_shift_pct=0.25),\n TimeStretch(p=0.25, max_rate=1.5, min_rate=0.5),\n])\n\n\n\nif os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n test_path1 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\nelse:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n test_path1 = './cmi-detect-behavior-with-sensor-data/test.csv'\n test_path2 = './cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n else:\n test_path1 = './test.csv'\n test_path2 = './test_demographics.csv'\n\nclass model_zhou_v26:\n def __init__(self, kaggle_input_path=\"/kaggle/input/cmi3v23\", seed=42, save_path='./'):\n self.models = []\n\n import os\n if os.path.exists(\"../input/cmi-detect-behavior-with-sensor-data\"):\n self.TRAIN = False \n self.RAW_DIR = Path(\"../input/cmi-detect-behavior-with-sensor-data\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n if os.path.exists(\"./cmi-detect-behavior-with-sensor-data/\"):\n self.TRAIN = True \n self.RAW_DIR = Path(\"./cmi-detect-behavior-with-sensor-data/\")\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n self.EXPORT_DIR = Path(save_path) \n else:\n self.TRAIN = True \n self.RAW_DIR = Path(\"./\")\n self.EXPORT_DIR = Path(save_path)\n self.PRETRAINED_DIR = Path(kaggle_input_path) \n\n if not os.path.exists(self.EXPORT_DIR):\n os.system(f'mkdir {self.EXPORT_DIR}')\n \n self.VALIDATION = False\n self.SEED = seed\n\n self.BATCH_SIZE = 64 * 1\n self.PAD_PERCENTILE = 128 + 64\n self.maxlen = self.PAD_PERCENTILE\n self.LR_INIT = 1e-3\n self.WD = 2e-1\n \n self.PATIENCE = 40\n self.random_state = self.SEED\n \n self.tof_mode = 4\n \n self.FOLDS = 5\n self.TRAIN_FOLDS = [0, 1, 2, 3, 4,]\n \n self.EPOCHS = 160\n\n self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"▶ imports ready · pytorch {torch.__version__} · device: {self.device}\")\n \n def create_model(self, param_a, param_b, param_c, param_d): \n def get_gravity(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = np.concatenate([g_x, g_y, g_z], -1)\n return gravity\n \n def rotate_quaternion_np(quat: np.ndarray, angle_range: tuple = [(-10, 10), (-10, 10), (-10, 10), ]) -> np.ndarray:\n # 确保输入形状正确\n if quat.ndim != 2 or quat.shape[1] != 4:\n raise ValueError(\"输入四元数必须是形状为[len, 4]的数组\")\n \n seq_len = quat.shape[0]\n \n # 1. 生成绕x、y、z轴的随机旋转角度（度）\n angle_x = np.random.uniform(angle_range[0][0], angle_range[0][1])\n angle_y = np.random.uniform(angle_range[1][0], angle_range[1][1])\n angle_z = np.random.uniform(angle_range[2][0], angle_range[2][1])\n \n # 2. 将角度转换为弧度\n rad_x = math.pi * angle_x / 180.0\n rad_y = math.pi * angle_y / 180.0\n rad_z = math.pi * angle_z / 180.0\n \n # 3. 生成绕各轴旋转的四元数\n qx = np.array([math.sin(rad_x/2), 0, 0, math.cos(rad_x/2)], dtype=np.float32) # x轴旋转\n qy = np.array([0, math.sin(rad_y/2), 0, math.cos(rad_y/2)], dtype=np.float32) # y轴旋转\n qz = np.array([0, 0, math.sin(rad_z/2), math.cos(rad_z/2)], dtype=np.float32) # z轴旋转\n \n # 4. 组合旋转四元数 (q_total = qz * qy * qx，注意乘法顺序)\n q_zy = _quat_mul(qz, qy)\n q_total = _quat_mul(q_zy, qx)\n \n # 5. 将组合旋转应用到每个四元数上\n # 扩展旋转四元数以匹配序列长度\n q_total_expanded = np.tile(q_total, (seq_len, 1)) # [len, 4]\n rotated_quat = _quat_mul(q_total_expanded, quat)\n \n # 6. 归一化确保四元数有效性\n norms = np.linalg.norm(rotated_quat, axis=1, keepdims=True)\n rotated_quat = rotated_quat / np.maximum(norms, 1e-8) # 防止除零\n \n return rotated_quat\n \n def _quat_mul(q1: np.ndarray, q2: np.ndarray) -> np.ndarray:\n # 提取分量（利用广播机制自动适配维度）\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n # 四元数乘法公式（完全向量化）\n x = w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2\n y = w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2\n z = w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2\n w = w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2\n \n # 堆叠结果（保持维度）\n return np.stack([x, y, z, w], axis=-1).astype(q1.dtype)\n \n \n def remove_average_rotation_optimized(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 从四元数序列中移除平均旋转，忽略全零四元数，并旋转到指定方向\n Args:\n q: 输入四元数序列，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n eps: 数值稳定性阈值\n Returns:\n q_final: 移除平均旋转并旋转到[0.5,0.5,0.5,0.5]方向后的四元数序列，形状 [bs, len, 4]\n 原始全零的位置仍为零\n \"\"\"\n def is_zero_quaternion(q: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 判断四元数是否为全零（或接近全零）\n Args:\n q: 四元数，形状 [..., 4]\n eps: 接近零的阈值\n Returns:\n 布尔张量，形状 [...], 指示每个四元数是否为零\n \"\"\"\n return torch.norm(q, dim=-1) < eps\n \n def safe_normalize(q: torch.Tensor, eps: float = 1e-12) -> torch.Tensor:\n \"\"\"\n 安全地规范化四元数，避免除零错误\n Args:\n q: 四元数，形状 [..., 4]\n eps: 最小范数阈值\n Returns:\n 规范化后的四元数\n \"\"\"\n norm = torch.norm(q, dim=-1, keepdim=True)\n # 添加最小范数保护\n norm = torch.clamp(norm, min=eps)\n return q / norm\n \n def quat_multiply(q1: torch.Tensor, q2: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 批量四元数乘法 (q1 * q2)\n Args:\n q1: 形状 [..., 4]，四元数 [x1,y1,z1,w1]\n q2: 形状 [..., 4]，四元数 [x2,y2,z2,w2]\n Returns:\n q: 形状 [..., 4]，乘积 q1*q2 [x,y,z,w]\n \"\"\"\n x1, y1, z1, w1 = q1[..., 0], q1[..., 1], q1[..., 2], q1[..., 3]\n x2, y2, z2, w2 = q2[..., 0], q2[..., 1], q2[..., 2], q2[..., 3]\n \n x = w1*x2 + x1*w2 + y1*z2 - z1*y2\n y = w1*y2 - x1*z2 + y1*w2 + z1*x2\n z = w1*z2 + x1*y2 - y1*x2 + z1*w2\n w = w1*w2 - x1*x2 - y1*y2 - z1*z2\n \n return torch.stack([x, y, z, w], dim=-1)\n \n def find_first_valid_indices(mask: torch.Tensor) -> torch.Tensor:\n \"\"\"\n 向量化方法查找每个batch中第一个有效四元数的索引\n Args:\n mask: 有效掩码，形状 [bs, len]\n Returns:\n indices: 每个batch的第一个有效索引，形状 [bs]\n \"\"\"\n bs, seq_len = mask.shape\n device = mask.device\n \n # 创建索引矩阵\n indices = torch.arange(seq_len, device=device).expand(bs, seq_len)\n # 将无效位置的索引设置为大数\n indices = torch.where(mask, indices, seq_len)\n # 找到最小索引（第一个有效位置）\n first_valid = torch.min(indices, dim=1)[0]\n # 处理全无效的情况\n all_invalid = first_valid == seq_len\n first_valid[all_invalid] = 0 # 设置为0，后续会处理\n \n return first_valid\n \n def quaternion_average_frechet_optimized(q: torch.Tensor, mask: torch.Tensor, eps: float = 1e-8) -> torch.Tensor:\n \"\"\"\n 计算批量四元数的Frechet均值（优化版本）\n Args:\n q: 输入四元数，形状 [bs, len, 4]，顺序 [rot_x, rot_y, rot_z, rot_w]\n mask: 有效四元数掩码，形状 [bs, len]，1表示有效，0表示无效\n eps: 数值稳定性阈值\n Returns:\n q_avg: 平均四元数，形状 [bs, 4]\n \"\"\"\n bs, len_q, _ = q.shape\n device = q.device\n \n # 步骤1：安全地单位化四元数\n q_normalized = safe_normalize(q, eps=eps)\n \n # 应用掩码：无效四元数置为零\n mask_keepdim = mask.unsqueeze(-1) # [bs, len, 1]\n q_normalized = q_normalized * mask_keepdim\n \n # 步骤2：向量化查找参考四元数\n first_valid_indices = find_first_valid_indices(mask)\n q_ref = q_normalized[torch.arange(bs, device=device), first_valid_indices].unsqueeze(1) # [bs, 1, 4]\n \n # 步骤3：改进的符号统一策略（添加阈值避免在接近正交时翻转）\n dot = (q_normalized * q_ref).sum(dim=-1, keepdim=True) # [bs, len, 1]\n q_unified = torch.where(dot < -0.01, -q_normalized, q_normalized)\n \n # 步骤4：构建协方差矩阵 M [bs, 4, 4]，考虑掩码\n q_reshaped = q_unified.unsqueeze(-1) # [bs, len, 4, 1]\n q_transposed = q_reshaped.transpose(2, 3) # [bs, len, 1, 4]\n outer_products = q_reshaped @ q_transposed # [bs, len, 4, 4]\n \n # 应用掩码并求和\n mask_4d = mask.unsqueeze(-1).unsqueeze(-1) # [bs, len, 1, 1]\n outer_products = outer_products * mask_4d\n M = outer_products.sum(dim=1) # [bs, 4, 4]\n \n # 处理所有四元数都无效的情况\n valid_count = mask.sum(dim=1) # [bs]\n all_invalid = valid_count == 0\n if all_invalid.any():\n # 对全无效的batch，使用单位四元数作为平均\n M[all_invalid] = torch.eye(4, device=device)\n \n # 步骤5：使用更稳定的特征值求解\n # 添加小单位矩阵增强数值稳定性\n M_stable = M + eps * torch.eye(4, device=device).unsqueeze(0)\n w, v = torch.linalg.eigh(M_stable) # w: [bs,4], v: [bs,4,4]\n q_avg = v[:, :, -1] # 最大特征值对应特征向量\n \n # 确保单位化 + 实部(rot_w)为正\n q_avg = safe_normalize(q_avg, eps=eps)\n q_avg = torch.where(q_avg[:, 3:4] < 0, -q_avg, q_avg)\n \n return q_avg\n \n bs, len_q, _ = q.shape\n device = q.device\n \n # 1. 识别全零四元数\n zero_mask = is_zero_quaternion(q, eps) # [bs, len]\n valid_mask = ~zero_mask # [bs, len]，1表示有效四元数\n \n # 2. 计算有效四元数的平均旋转\n q_avg = quaternion_average_frechet_optimized(q, valid_mask, eps) # [bs, 4]\n \n # 3. 创建目标四元数 [0.5, 0.5, 0.5, 0.5] 并规范化\n target_quat = torch.tensor([0, 0, 0, 1], device=device, dtype=q.dtype)\n target_quat = safe_normalize(target_quat)\n \n # 4. 将目标旋转整合到平均旋转中\n # 计算目标旋转与平均旋转的复合旋转\n # q_composite = target_quat * q_avg_conj\n # 这等价于先应用平均旋转的逆，然后应用目标旋转\n \n # 计算平均四元数的共轭（逆）\n q_avg_conj = q_avg.clone()\n q_avg_conj[:, :3] *= -1 # 虚部取反：[x,y,z,w] → [-x,-y,-z,w]\n \n # 将目标旋转与平均旋转的共轭相乘\n q_composite = quat_multiply(\n target_quat.unsqueeze(0).expand(bs, 4), # 扩展目标四元数以匹配批次大小\n q_avg_conj\n ) # [bs, 4]\n \n q_composite = safe_normalize(q_composite) # 确保复合旋转是单位四元数\n q_composite = q_composite.unsqueeze(1) # [bs, 1, 4]，便于广播\n \n # 5. 应用复合旋转：q_final = q_composite * q\n q_final = quat_multiply(q_composite, q) # [bs, len, 4]\n \n # 6. 保持原始全零位置为零\n q_final = q_final * valid_mask.unsqueeze(-1) # [bs, len, 4]\n \n return q_final\n\n \n class MotionFeatureExtractor(nn.Module):\n \"\"\"\n 支持CUDA/CPU输入的运动特征提取模块\n 输入: [bs, len, 7] 张量（[acc_x, acc_y, acc_z, rot_x, rot_y, rot_z, rot_w]）\n 输出: [bs, len, 7] 张量（[线性加速度x/y/z, 角速度x/y/z, 角距离]）\n \"\"\"\n def __init__(self, time_delta=1/200):\n super().__init__()\n self.time_delta = time_delta\n # ！修改1：不提前固定gravity_world的设备，改为使用时动态匹配输入设备\n self.gravity_world_val = torch.tensor([0.0, 0.0, 9.81], dtype=torch.float32)\n \n def quat_to_rot_matrix(self, quat):\n \"\"\"四元数（[x,y,z,w]）转旋转矩阵 [bs, len, 3, 3]\"\"\"\n x, y, z, w = quat[..., 0], quat[..., 1], quat[..., 2], quat[..., 3]\n \n xx = x * x\n yy = y * y\n zz = z * z\n xy = x * y\n xz = x * z\n yz = y * z\n xw = x * w\n yw = y * w\n zw = z * w\n ww = w * w\n \n # 旋转矩阵计算（基于输入quat的设备，自动在CUDA/CPU上运算）\n rot_mat = torch.stack([\n xx + ww - (yy + zz), 2 * (xy - zw), 2 * (xz + yw),\n 2 * (xy + zw), yy + ww - (xx + zz), 2 * (yz - xw),\n 2 * (xz - yw), 2 * (yz + xw), zz + ww - (xx + yy)\n ], dim=-1).view(*quat.shape[:-1], 3, 3)\n \n return rot_mat\n \n def rot_matrix_inverse(self, rot_mat):\n \"\"\"旋转矩阵求逆（逆=转置，设备与输入一致）\"\"\"\n return rot_mat.transpose(-2, -1)\n \n def rot_matrix_multiply(self, rot_mat1, rot_mat2):\n \"\"\"旋转矩阵乘法（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat1, rot_mat2)\n \n def apply_rotation(self, rot_mat, vec):\n \"\"\"旋转矩阵应用于向量（设备自动匹配）\"\"\"\n return torch.matmul(rot_mat, vec.unsqueeze(-1)).squeeze(-1)\n \n def remove_gravity_from_acc(self, acc_data, rot_data):\n \"\"\"从加速度中移除重力（核心：设备动态匹配）\"\"\"\n bs, seq_len, _ = acc_data.shape\n device = acc_data.device # ！修改2：获取输入设备（CUDA/CPU）\n \n valid_quats_normalized = rot_data / torch.norm(rot_data, dim=-1, keepdim=True).clamp(min=1e-8)\n rotations = self.quat_to_rot_matrix(valid_quats_normalized)\n \n # ！修改3：gravity_world动态匹配输入设备，避免CPU/CUDA不兼容\n gravity_world = self.gravity_world_val.to(device).expand(bs, seq_len, 3)\n # 重力向量从世界坐标系转到传感器坐标系\n gravity_sensor_frame = self.apply_rotation(\n self.rot_matrix_inverse(rotations),\n gravity_world\n )\n \n # 线性加速度计算（设备与输入一致）\n linear_accel = acc_data - gravity_sensor_frame\n return linear_accel\n \n def calculate_angular_velocity_from_quat(self, rot_data):\n \"\"\"从四元数计算角速度（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改4：获取输入设备\n angular_vel = torch.zeros(bs, seq_len, 3, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_vel\n \n # 四元数有效性判断（torch.tensor(0.0)改为匹配设备）\n quat_norms = torch.norm(rot_data, dim=-1)\n # ！修改5：用rot_data.new_zeros(1)创建同设备的0张量，替代硬编码的torch.tensor(0.0)\n valid_mask = ~(torch.isnan(quat_norms) | torch.isclose(quat_norms, rot_data.new_zeros(1)))\n valid_pairs_mask = valid_mask[:, :-1] & valid_mask[:, 1:]\n \n if torch.any(valid_pairs_mask):\n # 提取有效四元数对\n q_t = rot_data[:, :-1][valid_pairs_mask]\n q_t_plus_dt = rot_data[:, 1:][valid_pairs_mask]\n \n # 归一化\n q_t_norms = quat_norms[:, :-1][valid_pairs_mask].unsqueeze(-1)\n q_t_plus_dt_norms = quat_norms[:, 1:][valid_pairs_mask].unsqueeze(-1)\n q_t_norm = q_t / q_t_norms.clamp(min=1e-8)\n q_t_plus_dt_norm = q_t_plus_dt / q_t_plus_dt_norms.clamp(min=1e-8)\n \n # 旋转矩阵与相对旋转计算\n rot_t = self.quat_to_rot_matrix(q_t_norm)\n rot_t_plus_dt = self.quat_to_rot_matrix(q_t_plus_dt_norm)\n delta_rot = self.rot_matrix_multiply(self.rot_matrix_inverse(rot_t), rot_t_plus_dt)\n \n # 罗德里格斯公式求旋转向量（设备自动匹配）\n trace = delta_rot[..., 0, 0] + delta_rot[..., 1, 1] + delta_rot[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n sin_theta = torch.sin(theta)\n \n # 旋转向量计算（零张量指定设备）\n rot_vec = torch.zeros_like(q_t_norm[..., :3], device=device)\n # ！修改6：同修改5，避免设备不兼容\n mask = ~torch.isclose(sin_theta, rot_data.new_zeros(1))\n \n if torch.any(mask):\n factor = theta / (2 * sin_theta)\n rot_vec[mask] = factor[mask, None] * torch.stack([\n delta_rot[mask, 2, 1] - delta_rot[mask, 1, 2],\n delta_rot[mask, 0, 2] - delta_rot[mask, 2, 0],\n delta_rot[mask, 1, 0] - delta_rot[mask, 0, 1]\n ], dim=1)\n \n # 角速度计算\n angular_vel[:, :-1][valid_pairs_mask] = rot_vec / self.time_delta\n \n # 最后一个时间步复制前一个值\n if seq_len > 1:\n angular_vel[:, -1] = angular_vel[:, -2]\n \n return angular_vel\n \n def calculate_angular_distance(self, rot_data, cumulative=False):\n \"\"\"计算角距离（设备动态匹配）\"\"\"\n bs, seq_len, _ = rot_data.shape\n device = rot_data.device # ！修改7：获取输入设备\n angular_dist = torch.zeros(bs, seq_len, 1, device=device) # 零张量指定设备\n \n if seq_len < 2:\n return angular_dist\n \n # 四元数归一化\n quat_norms = torch.norm(rot_data, dim=-1)\n q1 = rot_data[:, :-1]\n q2 = rot_data[:, 1:]\n q1_norms = quat_norms[:, :-1].unsqueeze(-1)\n q2_norms = quat_norms[:, 1:].unsqueeze(-1)\n q1_norm = q1 / q1_norms.clamp(min=1e-8)\n q2_norm = q2 / q2_norms.clamp(min=1e-8)\n \n # 相对旋转与角距离计算\n r1 = self.quat_to_rot_matrix(q1_norm)\n r2 = self.quat_to_rot_matrix(q2_norm)\n relative_rotation = self.rot_matrix_multiply(self.rot_matrix_inverse(r1), r2)\n \n trace = relative_rotation[..., 0, 0] + relative_rotation[..., 1, 1] + relative_rotation[..., 2, 2]\n theta = torch.acos(torch.clamp((trace - 1) / 2, -1.0, 1.0))\n angular_dist[:, :-1, 0] = theta\n \n # 最后一个时间步处理\n if seq_len > 1:\n angular_dist[:, -1] = angular_dist[:, -2] if cumulative else 0.0\n \n # 累积角距离（设备一致）\n if cumulative:\n angular_dist = torch.cumsum(angular_dist, dim=1)\n \n return angular_dist\n \n def forward(self, x):\n \"\"\"前向传播（自动适配CUDA/CPU输入）\"\"\"\n # 分离加速度（前3维）和四元数（后4维）\n acc_data = x[..., :3]\n rot_data = x[..., 3:]\n \n # 计算三大特征（设备自动匹配输入）\n linear_accel = self.remove_gravity_from_acc(acc_data, rot_data)\n angular_vel = self.calculate_angular_velocity_from_quat(rot_data)\n angular_dist = self.calculate_angular_distance(rot_data)\n \n # 拼接输出（设备与输入一致）\n features = torch.cat([linear_accel, angular_vel, angular_dist], dim=-1)\n return features\n \n\n \n class ImuFeatureExtractor(nn.Module):\n def __init__(self, ):\n super().__init__()\n k = 15\n self.lpf_all = nn.Conv1d(7, 7, kernel_size=7, padding=7//2, groups=7, bias=False)\n \n self.lpf_acc = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n self.lpf_acc2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n self.lpf_gyro2 = nn.Conv1d(3, 3, k, padding=k//2, groups=3, bias=False)\n \n def forward(self, imu):\n # imu: \n B, C, T = imu.shape\n acc = imu[:, 0:3, :] # acc_x, acc_y, acc_z\n gyro = imu[:, 3:6, :] # gyro_x, gyro_y, gyro_z\n extra = imu[:, 6:, :]\n \n # 1) magnitude\n acc_mag = torch.norm(acc, dim=1, p=2, keepdim=True) # (B,1,T)\n gyro_mag = torch.norm(gyro, dim=1, p=2, keepdim=True)\n \n # 1.2) magnitude\n acc2 = acc/acc_mag.clip(1e-12)\n gyro2 = gyro/gyro_mag.clip(1e-12)\n \n \n acc_lpf2 = self.lpf_acc2(acc2)\n acc_hpf2 = acc2 - acc_lpf2\n gyro_lpf2 = self.lpf_gyro2(gyro2)\n gyro_hpf2 = gyro2 - gyro_lpf2\n \n \n # 1.3) magnitude\n acc_mag2 = torch.norm(acc, dim=1, p=1, keepdim=True) # (B,1,T)\n gyro_mag2 = torch.norm(gyro, dim=1, p=1, keepdim=True)\n \n # 1.4) magnitude\n acc3 = acc/acc_mag2.clip(1e-12)\n gyro3 = gyro/gyro_mag2.clip(1e-12)\n \n # 2) jerk \n jerk = F.pad(acc[:, :, 1:] - acc[:, :, :-1], (1,0)) # (B,3,T)\n gyro_delta = F.pad(gyro[:, :, 1:] - gyro[:, :, :-1], (1,0))\n \n # 2) jerk level2\n jerk2 = F.pad(acc[:, :, 2:] + acc[:, :, :-2] - acc[:, :, 1:-1] * 2, (1,1)) # (B,3,T)\n gyro_delta2 = F.pad(gyro[:, :, 2:] + gyro[:, :, :-2] - gyro[:, :, 1:-1] * 2, (1,1))\n \n # 3) energy\n acc_pow = acc ** 2\n gyro_pow = gyro ** 2\n \n # 4) LPF / HPF \n acc_lpf = self.lpf_acc(acc)\n acc_hpf = acc - acc_lpf\n gyro_lpf = self.lpf_gyro(gyro)\n gyro_hpf = gyro - gyro_lpf\n \n \n imu_hpf = imu - self.lpf_all(imu)\n \n features = [\n acc, gyro,\n \n # acc2, gyro2,\n acc_mag, gyro_mag,\n \n acc3, gyro3,\n acc_mag2, gyro_mag2,\n \n jerk, gyro_delta,\n jerk2, gyro_delta2, \n \n acc_pow, gyro_pow,\n \n acc_lpf, acc_hpf,\n gyro_lpf, gyro_hpf,\n \n acc_lpf2, acc_hpf2,\n gyro_lpf2, gyro_hpf2,\n \n imu_hpf,\n \n extra, imu.argsort(-1).argsort(-1)/128,\n \n ]\n \n return torch.cat(features, dim=1) # (B, C_out, T)\n \n class SEBlock(nn.Module):\n def __init__(self, channels, reduction=8):\n super().__init__()\n self.squeeze = nn.AdaptiveAvgPool1d(1)\n self.excitation = nn.Sequential(\n nn.Linear(channels, channels // reduction, bias=False),\n nn.ReLU(inplace=True),\n nn.Linear(channels // reduction, channels, bias=False),\n nn.Sigmoid()\n )\n \n def forward(self, x):\n b, c, _ = x.size()\n y = self.squeeze(x).view(b, c)\n y = self.excitation(y).view(b, c, 1)\n return x * y.expand_as(x)\n\n class ResidualSECNNBlock(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3, weight_decay=1e-4):\n super().__init__()\n \n # First conv block\n self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn1 = nn.BatchNorm1d(out_channels)\n \n # Second conv block\n self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n self.bn2 = nn.BatchNorm1d(out_channels)\n \n # SE block\n self.se = SEBlock(out_channels)\n \n # Shortcut connection\n self.shortcut = nn.Sequential()\n if in_channels != out_channels:\n self.shortcut = nn.Sequential(\n nn.Conv1d(in_channels, out_channels, 1, bias=False),\n nn.BatchNorm1d(out_channels)\n )\n self.pool_size = pool_size\n self.pool = nn.MaxPool1d(pool_size)\n self.dropout = nn.Dropout(dropout)\n \n self.act = nn.ReLU()\n\n self.gru = nn.GRU(out_channels, out_channels//2, batch_first=True, bidirectional=True)\n \n def forward(self, x):\n shortcut = self.shortcut(x)\n \n # First conv\n out = self.act(self.bn1(self.conv1(x)))\n # Second conv\n out = self.bn2(self.conv2(out))\n \n # SE block\n out = self.se(out)\n \n # Add shortcut\n \n out += shortcut\n \n out = out + self.gru(out.transpose(1, 2))[0].transpose(1, 2)\n out = self.act(out)\n\n out = self.act(out)\n \n \n # Pool and dropout\n if self.pool_size>1:\n out = self.pool(out)\n out = self.dropout(out)\n \n return out\n\n class AttentionLayer(nn.Module):\n def __init__(self, hidden_dim):\n super().__init__()\n self.attention = nn.Linear(hidden_dim, 1)\n \n def forward(self, x):\n # x shape: (batch, seq_len, hidden_dim)\n scores = torch.tanh(self.attention(x)) # (batch, seq_len, 1)\n weights = F.softmax(scores.squeeze(-1), dim=1) # (batch, seq_len)\n context = torch.sum(x * weights.unsqueeze(-1), dim=1) # (batch, hidden_dim)\n return context\n\n\n def get_gravity_torch(quat):\n g=9.81\n # 四元数旋转\n x = quat[..., 0:1] # rot_x\n y = quat[..., 1:2] # rot_y\n z = quat[..., 2:3] # rot_z\n w = quat[..., 3:4] # rot_w\n g_x = 2 * g * (x * z - w * y)\n g_y = 2 * g * (y * z + w * x)\n g_z = g * (w**2 - x**2 - y**2 + z**2)\n gravity = torch.concat([g_x, g_y, g_z], -1)\n return gravity\n\n tof_mode = self.tof_mode\n \n class TwoBranchModel(nn.Module):\n def __init__(self, pad_len, imu_dim_raw, tof_dim, n_classes, dropouts=[0.3, 0.3, 0.3, 0.3, 0.4, 0.5, 0.3], feature_engineering=True, **kwargs):\n super().__init__()\n self.imu_fe1 = ImuFeatureExtractor(**kwargs)\n self.imu_fe2 = ImuFeatureExtractor(**kwargs)\n self.imu_fe3 = ImuFeatureExtractor(**kwargs)\n \n imu_dim = 32 + 1 + 14 + 7 + 6 + 6 + 7 \n\n self.new_feat_module = MotionFeatureExtractor()\n \n self.imu_dim = imu_dim\n self.tof_dim = tof_dim\n \n self.fir_nchan = 7\n self.thm_nchan = 5\n self.tof_nchan = 5 * (5 + 3 * tof_mode)\n \n weight_decay = 3e-3\n \n \n # IMU deep branch\n self.imu_block1 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block2 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n \n self.imu_block12 = ResidualSECNNBlock(imu_dim * 1, 160, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n self.imu_block22 = ResidualSECNNBlock(160, 256, 5, dropout=dropouts[1], pool_size=1, weight_decay=weight_decay)\n\n self.thm_block = nn.Sequential(\n ResidualSECNNBlock(self.thm_nchan * 1, 32, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(32, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.tof_block = nn.Sequential(\n ResidualSECNNBlock(self.tof_nchan * 1, 64, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay),\n ResidualSECNNBlock(64, 256, 5, dropout=dropouts[0], pool_size=1, weight_decay=weight_decay)\n )\n \n self.emb_all = 256\n \n self.dropout1d = nn.Dropout1d(0.15)\n \n # BiLSTM\n self.dim_lstm = 512\n self.dim_encoder = self.dim_lstm * 2\n \n self.bilstm = nn.LSTM(self.emb_all, self.dim_lstm, num_layers=2, bidirectional=True, batch_first=True)\n \n self.lstm_dropout = nn.Dropout(dropouts[4])\n \n self.output2 = nn.Linear(self.dim_encoder, 5)\n \n # Dense layers\n \n self.dense1 = nn.Linear(self.dim_encoder * 2, 512, bias=False)\n self.bn_dense1 = nn.BatchNorm1d(512)\n self.drop1 = nn.Dropout(dropouts[5])\n \n self.dense2 = nn.Linear(512, 512, bias=False)\n self.bn_dense2 = nn.BatchNorm1d(512)\n self.drop2 = nn.Dropout(dropouts[6])\n \n self.output3 = nn.Linear(512, 5)\n self.classifier = nn.Linear(512, n_classes * 4)\n \n self.scale = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale2 = nn.Parameter(torch.ones((1, 256, 1)))\n self.bias2 = nn.Parameter(torch.zeros((1, 256, 1)))\n \n self.scale3 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias3 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.scale4 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias4 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale5 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias5 = nn.Parameter(torch.zeros((1, 512, 1)))\n\n self.scale6 = nn.Parameter(torch.ones((1, 512, 1)))\n self.bias6 = nn.Parameter(torch.zeros((1, 512, 1)))\n \n self.act = nn.GELU()\n \n def forward(self, x, ):\n # Split input\n mask_all = (x.abs().max(-1)[0]!=0).float()[:,:,None]\n mask_all_trans = mask_all.transpose(1, 2)\n \n imu = x[:, :, :self.fir_nchan] # (batch, imu_dim, seq_len)\n imu2 = self.new_feat_module(imu)\n imu2 = imu2[:,:,:7]\n\n imu = imu.transpose(1, 2)\n imu2 = imu2.transpose(1, 2)\n \n thm = x[:, :, self.fir_nchan:self.fir_nchan + self.thm_nchan].transpose(1, 2) # (batch, thm_dim, seq_len)\n tof = x[:, :, self.fir_nchan + self.thm_nchan:].transpose(1, 2) # (batch, tof_dim, seq_len)\n \n imu = self.imu_fe1(imu) # (B, imu_dim, T)\n imu2 = self.imu_fe2(imu2) # (B, imu_dim, T)\n \n imu = (imu * self.scale[:,:imu.shape[1],:] + self.bias[:,:imu.shape[1],:]) * mask_all.transpose(1, 2)\n imu2 = (imu2 * self.scale2[:,:imu2.shape[1],:] + self.bias2[:,:imu2.shape[1],:]) * mask_all.transpose(1, 2)\n \n thm = (thm * self.scale5[:,:thm.shape[1],:] + self.bias5[:,:thm.shape[1],:] ) * mask_all.transpose(1, 2)\n tof = (tof * self.scale6[:,:tof.shape[1],:] + self.bias6[:,:tof.shape[1],:] ) * mask_all.transpose(1, 2)\n \n thm = self.dropout1d(thm)\n tof = self.dropout1d(tof)\n imu = self.dropout1d(imu)\n imu2 = self.dropout1d(imu2)\n \n thm = self.thm_block(thm)\n tof = self.tof_block(tof)\n \n \n # IMU branch\n x1 = self.imu_block1(imu)\n x1 = self.imu_block2(x1)\n \n x12 = self.imu_block12(imu2)\n x12 = self.imu_block22(x12)\n \n x1 = (x1 + x12 + thm + tof)\n \n merged = x1.transpose(1, 2) # (batch, seq_len, 256)\n \n # BiLSTM\n lstm_out, _ = self.bilstm(merged)\n \n logits2 = self.output2(self.lstm_dropout(lstm_out))\n \n attended = torch.cat(((lstm_out).max(1)[0], (lstm_out).mean(1), ), -1)\n attended = self.lstm_dropout(attended)\n \n \n # Dense layers\n x = self.act(self.bn_dense1(self.dense1(attended)))\n x = self.drop1(x)\n x = self.act(self.bn_dense2(self.dense2(x)))\n x = self.drop2(x)\n \n # Classification\n logits = (self.classifier(x))\n logits3 = (self.output3(x))\n \n if not self.training:\n return logits.reshape(logits.shape[0], 4, 18).mean(1), logits3\n \n return logits, logits2, logits3\n\n\n return TwoBranchModel(param_a, param_b, param_c, param_d)\n\n def preprocess_sequence(self, df_seq: pd.DataFrame, feature_cols: list, scaler: StandardScaler):\n \"\"\"Normalizes and cleans the time series sequence\"\"\"\n mat = df_seq[feature_cols].ffill().bfill().fillna(0).values\n return (mat).astype('float32')\n \n def pad_sequences_torch(self, sequences, maxlen, padding='pre', truncating='pre', value=0.0):\n \"\"\"PyTorch equivalent of Keras pad_sequences\"\"\"\n result = []\n for seq in sequences:\n if len(seq) >= maxlen:\n if truncating == 'post':\n seq = seq[:maxlen]\n else: # 'pre'\n seq = seq[-maxlen:]\n else:\n pad_len = maxlen - len(seq)\n if padding == 'post':\n seq = np.concatenate([seq, np.full((pad_len, seq.shape[1]), value)])\n else: # 'pre'\n seq = np.concatenate([np.full((pad_len, seq.shape[1]), value), seq])\n result.append(seq)\n return np.array(result, dtype=np.float32)\n \n def preprocess_left_handed(self, l_tr):\n def handle_quaternion_missing_values(rot_data: np.ndarray) -> np.ndarray:\n \"\"\"\n Handle missing values in quaternion data intelligently\n \n Key insight: Quaternions must have unit length |q| = 1\n If one component is missing, we can reconstruct it from the others\n \"\"\"\n rot_cleaned = rot_data.copy()\n \n for i in range(len(rot_data)):\n row = rot_data[i]\n missing_count = np.isnan(row).sum()\n \n if missing_count == 0:\n # No missing values, normalize to unit quaternion\n norm = np.linalg.norm(row)\n if norm > 1e-8:\n rot_cleaned[i] = row / norm\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0] # Identity quaternion\n \n elif missing_count == 1:\n # One missing value, reconstruct using unit quaternion constraint\n # |w|² + |x|² + |y|² + |z|² = 1\n missing_idx = np.where(np.isnan(row))[0][0]\n valid_values = row[~np.isnan(row)]\n \n sum_squares = np.sum(valid_values**2)\n if sum_squares <= 1.0:\n missing_value = np.sqrt(max(0, 1.0 - sum_squares))\n # Choose sign for continuity with previous quaternion\n if i > 0 and not np.isnan(rot_cleaned[i-1, missing_idx]):\n if rot_cleaned[i-1, missing_idx] < 0:\n missing_value = -missing_value\n rot_cleaned[i, missing_idx] = missing_value\n rot_cleaned[i, ~np.isnan(row)] = valid_values\n else:\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n else:\n # More than one missing value, use identity quaternion\n rot_cleaned[i] = [0.0, 0.0, 0.0, 0.0]\n \n return rot_cleaned\n \n rot_cleaned = handle_quaternion_missing_values(l_tr[[\"rot_w\",\"rot_x\", \"rot_y\", \"rot_z\"]].to_numpy())\n rot_scipy = rot_cleaned[:, [1, 2, 3, 0]]\n \n norms = np.linalg.norm(rot_scipy, axis=1)\n if np.any(norms < 1e-8):\n # Replace problematic quaternions with identity\n mask = norms < 1e-8\n rot_scipy[mask] = [0.0, 0.0, 0.0, 1.0] # Identity quaternion in scipy format\n \n r = R.from_quat(rot_scipy)\n tmp = r.as_euler(\"xyz\")\n tmp[:,1] = - tmp[:,1]\n tmp[:,2] = - tmp[:,2] \n r = R.from_euler(\"xyz\", tmp)\n tmp = r.as_quat()\n \n if np.any(norms < 1e-8):\n mask = norms < 1e-8\n tmp[mask] = [0.0, 0.0, 0.0, 0.0]\n \n l_tr = l_tr.with_columns(pl.DataFrame(tmp, schema=[\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]))\n l_tr = l_tr.with_columns(-pl.col(\"acc_x\"))\n \n tmp = l_tr[[\"thm_3\", \"thm_5\"]]\n tmp.columns = [\"thm_5\", \"thm_3\"]\n l_tr = l_tr.with_columns(tmp)\n \n swap_1_2_4_base = [[0,7],[1,6],[2,5],[3,4], [4,3], [5,2],[6,1],[7,0]]\n swap_3_5_base = [[0,56],[8,48],[16,40], [24,32],[32,24], [40,16],[48,8], [56,0]]\n \n swap_1_2_4 = list()\n for i in range(0,64,8):\n ll = list()\n for (k,l) in swap_1_2_4_base:\n ll.append([k+i, l+i])\n swap_1_2_4 += ll\n \n swap_3_5 = list()\n for i in range(8):\n ll = list()\n for (k,l) in swap_3_5_base:\n ll.append([k+i, l+i])\n swap_3_5 += ll\n \n l_df = l_tr\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[k].alias(l))\n \n for (k,l) in zip([\"tof_3_v\" + str(x) for x in range(64)], [\"tof_5_v\" + str(x) for x in range(64)]):\n l_tr = l_tr.with_columns(l_df[l].alias(k))\n \n l_df = l_tr\n \n for i in [1,2,4]:\n for (k, l) in swap_1_2_4:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n \n for i in [3,5]:\n for (k, l) in swap_3_5:\n l_tr = l_tr.with_columns(l_df[\"tof_\" + str(i) + \"_v\"+str(k)].alias(\"tof_\" + str(i) + \"_v\"+str(l)))\n return l_tr\n \n def train(self, ):\n class CMI3Dataset(Dataset):\n def __init__(self,\n X_list,\n y_list, y_list2, y_list3,\n maxlen,\n mode=\"train\",\n imu_dim=7,\n augment=None,\n epoch_multiplier=1,\n ):\n self.X_list = X_list\n self.mode = mode\n self.y_list = y_list\n self.y_list2 = y_list2\n self.y_list3 = y_list3\n self.maxlen = maxlen\n self.imu_dim = imu_dim \n self.augment = augment\n self.epoch_multiplier = epoch_multiplier\n \n def __getitem__(self, index):\n X = self.X_list[index//self.epoch_multiplier]\n y = self.y_list[index//self.epoch_multiplier]\n y2 = self.y_list2[index//self.epoch_multiplier]\n y3 = self.y_list3[index//self.epoch_multiplier]\n\n if self.mode=='train':\n X_ = np.concatenate([X, y2], -1)\n \n X_ = transforms_custom(X_)\n \n X = X_[:,:X.shape[-1]]\n y2 = X_[:,X.shape[-1]:]\n \n X[:,3:7] = X[:,3:7]/(np.sqrt((X[:,3:7]**2).sum(-1))+1e-6)[:,None]\n \n return X, y, y2, y3\n \n def __len__(self):\n return len(self.X_list) * self.epoch_multiplier\n\n class EMA:\n def __init__(self, model, decay=0.999):\n self.decay = decay\n self.shadow = {}\n self.backup = {}\n \n for name, param in model.named_parameters():\n if param.requires_grad:\n self.shadow[name] = param.data.clone()\n \n def update(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad:\n assert name in self.shadow\n new_average = (1.0 - self.decay) * param.data + self.decay * self.shadow[name]\n self.shadow[name] = new_average.clone()\n \n def apply_shadow(self, model):\n self.backup = {}\n for name, param in model.named_parameters():\n if param.requires_grad:\n self.backup[name] = param.data.clone()\n param.data = self.shadow[name]\n \n def restore(self, model):\n for name, param in model.named_parameters():\n if param.requires_grad and name in self.backup:\n param.data = self.backup[name]\n self.backup = {}\n\n def set_seed(seed: int = 42):\n import numpy as np\n \n random.seed(seed)\n \n os.environ['PYTHONHASHSEED'] = str(seed)\n \n np.random.seed(seed)\n \n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.cuda.manual_seed_all(seed) \n # torch.backends.cudnn.deterministic = True\n # torch.backends.cudnn.benchmark = False\n # torch.use_deterministic_algorithms(True)\n \n set_seed(self.SEED)\n\n class RDROPLoss(nn.Module):\n \"\"\"\n RDROP损失函数实现\n 结合原始损失和KL散度约束\n \"\"\"\n def __init__(self, alpha=0.5):\n super(RDROPLoss, self).__init__()\n self.alpha = alpha # KL损失的权重系数\n self.kl_div = nn.KLDivLoss(reduction='batchmean')\n \n def forward(self, logits1, logits2):\n # KL散度约束：两次次输出分布的一致性\n p1 = F.log_softmax(logits1, dim=1)\n p2 = F.softmax(logits2, dim=1)\n kl_loss1 = self.kl_div(p1, p2)\n \n p2 = F.log_softmax(logits2, dim=1)\n p1 = F.softmax(logits1, dim=1)\n kl_loss2 = self.kl_div(p2, p1)\n \n # 总损失 = 分类损失 + alpha * KL散度损失\n total_loss = self.alpha * (kl_loss1 + kl_loss2)\n return total_loss\n \n import numpy as np\n import pandas as pd\n from scipy.spatial.transform import Rotation as R\n from tqdm.auto import tqdm \n \n print(\"▶ TRAIN MODE – loading dataset …\")\n df = pd.read_csv(self.RAW_DIR / \"train.csv\")\n self.demo = pd.read_csv(self.RAW_DIR / \"train_demographics.csv\")\n\n\n\n \n group_list = []\n for _, group in tqdm(df.groupby('sequence_id')):\n index_old = group.index\n if self.demo[self.demo['subject']==group['subject'].values[0]]['handedness'].values[0] ==0:\n ###################################################################################\n group = self.preprocess_left_handed(pl.from_pandas(group)).to_pandas()\n group.index = index_old\n \n group_list.append(group)\n df.update(pd.concat(group_list))\n\n\n \n \n dict_ = dict(zip(df['subject'].unique(), list(range(df['subject'].nunique()))))\n df['subject_id_new'] = df['subject'].map(dict_)\n \n #df = df[~df['subject'].isin({'SUBJ_045235', 'SUBJ_019262'})]\n \n dict_behavior = {'Relaxes and moves hand to target location': 1,\n 'Hand at target location': 2,\n 'Performs gesture': 3,\n 'Moves hand to target location': 4}\n \n dict_orientation = {\n 'Lie on Back': 0,\n 'Lie on Side - Non Dominant': 1,\n 'Seated Lean Non Dom - FACE DOWN': 2,\n 'Seated Straight': 3,\n }\n \n df['bid'] = df['behavior'].map(dict_behavior)\n df['oid'] = df['orientation'].map(dict_orientation)\n \n df, new_feat_cols, new_feat_cols2 = self.get_new_features(df)\n \n # Label encoding\n le = LabelEncoder()\n df['gesture_int'] = le.fit_transform(df['gesture'])\n\n\n df['gesture_int'] = df['gesture_int'] + df['oid'] * 18\n\n \n np.save(self.EXPORT_DIR / \"gesture_classes.npy\", le.classes_)\n \n # Feature list\n meta_cols = {'gesture', 'gesture_int', 'sequence_type', 'behavior', 'orientation',\n 'row_id', 'subject', 'phase', 'sequence_id', 'sequence_counter', 'subject_id_new'}\n feature_cols = [c for c in df.columns if c not in meta_cols]\n imu_cols = [c for c in feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n # tof_cols = [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n thm_cols = [c for c in feature_cols if c.startswith('thm_')]\n print(thm_cols)\n \n tof_cols = [] # [c for c in feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n \n imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w',]\n \n feature_cols = imu_cols + new_feat_cols + thm_cols + new_feat_cols2\n \n print(f\" IMU {len(imu_cols)+len(new_feat_cols)} | THM {len(thm_cols)} | TOF {len(new_feat_cols2)} | total {len(feature_cols)} features\")\n \n # Save feature_cols\n np.save(self.EXPORT_DIR / \"feature_cols.npy\", np.array(feature_cols))\n pad_len = self.PAD_PERCENTILE\n \n # Group sequences\n seq_gp = df.groupby('sequence_id')\n X_list_raw, y_list, id_list, subject_list, y2list, y3list = [], [], [], [], [], []\n for seq_id, seq in seq_gp:\n mat = seq[feature_cols].ffill().bfill().fillna(0).values\n X_list_raw.append(mat)\n y_list.append(seq['gesture_int'].iloc[0])\n id_list.append(seq_id)\n subject_list.append(seq['subject_id_new'].iloc[0])\n \n if len(seq) < pad_len:\n bid = np.zeros(pad_len)\n bid[-len(seq):] = seq['bid'].values.ravel()\n else:\n bid = seq['bid'].values.ravel()[-pad_len:]\n \n y2list.append(bid)\n y3list.append(seq['oid'].iloc[0])\n \n pad_len = self.PAD_PERCENTILE\n np.save(self.EXPORT_DIR / \"sequence_maxlen.npy\", pad_len)\n \n id_list = np.array(id_list)\n y_list_all = np.eye(4 * len(le.classes_))[y_list].astype(np.float32) # one-hot\n \n y_list_all2 = np.vstack(y2list).astype(int)\n y_list_all2 = (np.eye(5)[y_list_all2.reshape(-1, 1)]).reshape(-1, pad_len, 5).astype(np.float32)\n y_list_all3 = np.eye(5)[y3list].astype(np.float32)\n \n augmenter = None\n metrics = []\n \n criterion_rdrop = RDROPLoss(alpha=0.5)\n\n from sklearn.model_selection import GroupKFold\n gkf = GroupKFold(\n n_splits=self.FOLDS, \n shuffle=True, \n random_state=self.random_state\n )\n \n def clipped_cross_entropy(logits, y, clipval=0.6, num=18):\n return -torch.sum(F.log_softmax(logits, dim=1) * y.clip((1-clipval)/num, clipval), dim=1).mean() \n\n # return 1 - (logits.softmax(-1) * y).mean()\n \n idlistall = []\n targetfinalall = []\n predimuonlyall = []\n predallfeatall = []\n predorientationall = []\n targetsorientations = []\n foldlistall = []\n for fold, (train_idx, val_idx) in enumerate(gkf.split(id_list, id_list, groups=subject_list)):\n \n # skf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=self.random_state)\n # for fold, (train_idx, val_idx) in enumerate(skf.split(id_list, np.argmax(y_list_all, axis=1))): \n \n if fold not in self.TRAIN_FOLDS:\n continue\n \n print(f\"\\n▶ Fold {fold}\")\n \n X_list_scaled = [x for x in X_list_raw]\n X_list_all = self.pad_sequences_torch(X_list_scaled, maxlen=pad_len, padding='pre', truncating='pre')\n \n # Prepare train/val sets\n train_list = X_list_all[train_idx]\n train_y_list = y_list_all[train_idx]\n train_y_list2 = y_list_all2[train_idx]\n train_y_list3 = y_list_all3[train_idx]\n \n \n val_list = X_list_all[val_idx]\n val_y_list = y_list_all[val_idx]\n \n val_y_list2 = y_list_all2[val_idx]\n val_y_list3 = y_list_all3[val_idx]\n\n id_list_valid = id_list[val_idx]\n\n print(len(id_list_valid))\n idlistall.append(id_list_valid)\n foldlistall.append([fold for _ in range(len(id_list_valid))])\n \n # Data loaders\n train_dataset = CMI3Dataset(train_list, train_y_list, train_y_list2, train_y_list3, pad_len, mode=\"train\", imu_dim=len(imu_cols),\n augment=augmenter)\n train_loader = DataLoader(train_dataset, batch_size=self.BATCH_SIZE, shuffle=True, num_workers=6, drop_last=False, pin_memory=True)\n \n val_dataset = CMI3Dataset(val_list, val_y_list, val_y_list2, val_y_list3, pad_len, mode=\"val\")\n val_loader = DataLoader(val_dataset, batch_size=self.BATCH_SIZE, shuffle=False, num_workers=6, drop_last=False, pin_memory=True)\n\n device = self.device\n \n # Model & EMA \n model = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n model2 = self.create_model(pad_len, len(imu_cols), len(tof_cols), len(le.classes_)).to(device)\n\n if self.VALIDATION==True:\n checkpoint = torch.load(self.PRETRAINED_DIR / f'gesture_two_branch_fold{fold}.pth', map_location=device)\n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n \n ema = EMA(model, decay=0.998)\n \n # Optimizer\n hidden_weights = [p for p in model.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases = [p for p in model.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups = [\n dict(params=hidden_weights, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer = SingleDeviceMuonWithAuxAdam(param_groups)\n\n hidden_weights2 = [p for p in model2.parameters() if p.ndim >= 2 and p.requires_grad]\n hidden_gains_biases2 = [p for p in model2.parameters() if p.ndim < 2 and p.requires_grad]\n param_groups2 = [\n dict(params=hidden_weights2, use_muon=True, lr=0.005, weight_decay=self.WD),\n dict(params=hidden_gains_biases2, use_muon=False, lr=self.LR_INIT, betas=(0.9, 0.95), weight_decay=self.WD),\n ]\n optimizer2 = SingleDeviceMuonWithAuxAdam(param_groups2)\n \n nbatch = len(train_loader)\n nsteps = self.EPOCHS * nbatch\n \n scheduler = ConstantCosineLR(optimizer, nsteps, 0.) \n print(\"▶ Starting training...\")\n \n best_val_acc = 0\n for epoch in tqdm(range(self.EPOCHS)):\n model.train()\n model2.train()\n \n train_preds, train_targets = [], []\n train_loss = 0.0\n \n for X, y, y2, y3 in train_loader:\n if self.VALIDATION==True:\n break\n \n BS = X.shape[0]\n X0 = X.clone()\n\n X[BS//8:,:,7:] = 0.\n\n if np.random.random() > 0.5:\n X[:BS//8*2,:,3:7] = 0.\n else:\n X[:BS//8*2,:,:3] = 0.\n \n X, y = X.float().to(device), y.to(device)\n y2, y3 = y2.float().to(device), y3.to(device)\n X0 = X0.float().to(device)\n \n optimizer.zero_grad()\n optimizer2.zero_grad()\n \n LEN = 1200 + np.random.randint(80)\n \n logits, logits2, logits3 = model(X[:,-LEN:])\n \n logits_, logits2_, logits3_ = model2(X0[:,-LEN:])\n \n clipval = 0.6\n loss = clipped_cross_entropy(logits, y, clipval, 18) \n loss += clipped_cross_entropy(logits2, y2[:,-LEN:], clipval, 5) * 0.5\n loss += clipped_cross_entropy(logits3, y3, clipval, 5) * 0.5\n\n clipval = 0.6\n loss2 = clipped_cross_entropy(logits_, y, clipval, 18) \n loss2 += clipped_cross_entropy(logits2_, y2[:,-LEN:], clipval, 5) * 0.5\n loss2 += clipped_cross_entropy(logits3_, y3, clipval, 5) * 0.5\n\n if epoch > 40:\n loss = loss + loss2 + criterion_rdrop(logits, logits_)\n else:\n loss = loss + loss2 \n \n loss.backward()\n \n optimizer.step()\n optimizer2.step()\n \n ema.update(model)\n \n train_preds.append(logits.argmax(dim=1).cpu().numpy())\n train_targets.append(y.argmax(dim=1).cpu().numpy())\n \n scheduler.step()\n train_loss += loss.item()\n\n model2.eval()\n model.eval()\n ema.apply_shadow(model)\n \n val_loss = 0.0\n val_preds_accmask, val_targets_accmask = [], []\n val_preds_imuonly, val_targets_imuonly = [], []\n val_preds_rotmask, val_targets_rotmask = [], []\n\n val_preds_mother, val_targets_rotmask = [], []\n\n val_preds_logits = []\n val_preds_imuonly_logits = []\n\n val_preds_orientation, val_targets_orientation = [], []\n val_preds_orientation_logits = []\n \n \n with torch.inference_mode():\n for X, y, y2, y3 in val_loader:\n X_imuonly = X[:].clone()\n X_imuonly[:, :, 7:] = 0.0\n\n X_accmask = X[:].clone()\n X_accmask[:, :, 7:] = 0.0\n X_accmask[:, :, 0:3] = 0.0\n\n X_rotmask = X[:].clone()\n X_rotmask[:, :, 7:] = 0.0\n X_rotmask[:, :, 3:7] = 0.0\n \n X, y = X.float().to(device), y.to(device)\n X_imuonly = X_imuonly.float().to(device)\n X_accmask = X_accmask.float().to(device)\n X_rotmask = X_rotmask.float().to(device)\n\n logits_mother, logits_orientation = model(X)\n \n logits_rotmask, _ = model(X_rotmask)\n logits_accmask, _ = model(X_accmask)\n logits_imuonly, _ = model(X_imuonly)\n\n val_preds_mother.append(logits_mother.argmax(dim=1).cpu().numpy())\n val_preds_accmask.append(logits_accmask.argmax(dim=1).cpu().numpy())\n val_preds_rotmask.append(logits_rotmask.argmax(dim=1).cpu().numpy())\n val_preds_imuonly.append(logits_imuonly.argmax(dim=1).cpu().numpy())\n \n val_preds_logits.append(logits_mother.cpu().numpy())\n val_preds_imuonly_logits.append(logits_imuonly.cpu().numpy())\n \n val_targets_imuonly.append(y.reshape(y.shape[0], 4, 18).sum(1).argmax(dim=1).cpu().numpy())\n\n\n val_preds_orientation_logits.append(logits_orientation.cpu().numpy())\n val_preds_orientation.append(logits_orientation.argmax(dim=1).cpu().numpy())\n val_targets_orientation.append(y3.argmax(dim=1).cpu().numpy())\n \n \n if len(train_targets) >= 0:\n train_acc = 0.\n try:\n train_acc = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_targets)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(train_preds)]})\n )\n except:\n pass\n val_acc_rotmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_rotmask)]})\n )\n val_acc_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_imuonly)]})\n )\n val_acc_accmask = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_accmask)]})\n )\n val_acc_mother = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_targets_imuonly)]}),\n pd.DataFrame({'gesture': le.classes_[np.concatenate(val_preds_mother)]})\n )\n\n val_acc_orientation = np.mean(\n np.concatenate(val_preds_orientation)==np.concatenate(val_targets_orientation)\n )\n \n train_loss = np.mean(train_loss)\n print('epoch', epoch, 'loss : ', round(train_loss, 4), '| TRAIN : ', round(train_acc, 4), '| ORIENTATION: ', round(val_acc_orientation, 4), '| IMUONLY : ', round(val_acc_imuonly, 4), '| ROTMASK : ', round(val_acc_rotmask, 4), '| ACCMASK : ', round(val_acc_accmask, 4), '| MOTHER : ', round(val_acc_mother, 4), '| LR : ', optimizer.param_groups[0]['lr'])\n \n metric = val_acc_orientation # val_acc\n \n if metric > best_val_acc:\n best_val_acc = metric\n # Save model\n torch.save({\n 'model_state_dict': model.state_dict(),\n 'imu_dim': len(imu_cols),\n 'tof_dim': len(tof_cols),\n 'n_classes': len(le.classes_),\n 'pad_len': pad_len,\n \n }, self.EXPORT_DIR / f\"gesture_two_branch_fold{fold}.pth\")\n print(f\"fold: {fold} val_all_acc: {metric:.4f}\")\n print(\"✔ Training done – artefacts saved in\", self.EXPORT_DIR)\n \n ema.restore(model)\n\n targetfinalall.append(np.concatenate(val_targets_imuonly))\n predimuonlyall.append(np.concatenate(val_preds_imuonly_logits, 0))\n predallfeatall.append(np.concatenate(val_preds_logits, 0))\n predorientationall.append(np.concatenate(val_preds_orientation_logits, 0))\n targetsorientations.append(np.concatenate(val_targets_orientation))\n if self.VALIDATION:\n break\n \n \n ema.apply_shadow(model)\n metrics.append(best_val_acc)\n\n print(metrics, sum(metrics)/len(metrics))\n\n import joblib\n name = str(self.PRETRAINED_DIR).replace('/', '_')\n joblib.dump(\n {\n 'guestures': le.classes_, \n 'pred_all': np.concatenate(predallfeatall, 0), \n 'pred_imuonly': np.concatenate(predimuonlyall, 0), \n 'targets': np.concatenate(targetfinalall), \n 'idlist': np.concatenate(idlistall),\n 'fold': np.concatenate(foldlistall),\n 'pred_orientation' : np.concatenate(predorientationall, 0),\n 'target_orientation' : np.concatenate(targetsorientations), \n }, \n f\"oof_{name}.joblib\", \n )\n\n return le.classes_, np.concatenate(predallfeatall, 0), np.concatenate(predimuonlyall, 0), np.concatenate(targetfinalall), np.concatenate(idlistall), np.concatenate(predorientationall, 0), np.concatenate(targetsorientations)\n \n \n def get_new_features(self, df):\n feature_names = []\n \n for col in ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']:\n df[col] = df[col] / 5. \n \n new_columns = {} \n for i in range(1, 6):\n pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n \n new_columns.update({\n f'tof_{i}_isna_mean1': (df[pixel_cols]==-1).mean(axis=1),\n f'tof_{i}_isna_mean2': 1 - (df[pixel_cols].isna()).mean(axis=1),\n })\n \n df[pixel_cols] = 6 - df[pixel_cols].replace(-1, 255.) / 50. \n \n tof_data = df[pixel_cols]\n \n new_columns.update({\n f'tof_{i}_mean': tof_data.mean(axis=1),\n f'tof_{i}_std': tof_data.std(axis=1),\n # f'tof_{i}_min': tof_data.min(axis=1),\n f'tof_{i}_max': tof_data.max(axis=1),\n \n # f'tof_{i}_median_norm': tof_data.median(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_max_norm': tof_data.min(axis=1)/tof_data.mean(axis=1).clip(1),\n # f'tof_{i}_min_norm': tof_data.max(axis=1)/tof_data.mean(axis=1).clip(1),\n })\n if self.tof_mode > 1:\n region_size = 64 // self.tof_mode\n for r in (range(self.tof_mode)):\n region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n new_columns.update({\n f'tof{self.tof_mode}_{i}_region_{r}_mean': region_data.mean(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_std': region_data.std(axis=1),\n # f'tof{self.tof_mode}_{i}_region_{r}_min': region_data.min(axis=1),\n f'tof{self.tof_mode}_{i}_region_{r}_max': region_data.max(axis=1),\n })\n \n df_tof = pd.DataFrame(new_columns)\n df = pd.concat([df, df_tof], axis=1)\n \n \n return df, feature_names, list(df_tof.columns)\n \n\n def get_model(self, ): \n print(\"▶ INFERENCE MODE – loading artefacts from\", self.PRETRAINED_DIR)\n self.feature_cols = np.load(self.PRETRAINED_DIR / \"feature_cols.npy\", allow_pickle=True).tolist()\n self.pad_len = int(np.load(self.PRETRAINED_DIR / \"sequence_maxlen.npy\"))\n self.gesture_classes = np.load(self.PRETRAINED_DIR / \"gesture_classes.npy\", allow_pickle=True)\n \n self.imu_cols = [c for c in self.feature_cols if not (c.startswith('thm_') or c.startswith('tof_'))]\n self.tof_cols = [c for c in self.feature_cols if c.startswith('thm_') or c.startswith('tof_')]\n # Load model\n MODELS = [f'gesture_two_branch_fold{i}.pth' for i in range(self.FOLDS) if i in self.TRAIN_FOLDS]\n \n self.models = []\n for path in MODELS:\n checkpoint = torch.load(self.PRETRAINED_DIR / path, map_location=self.device)\n \n model = self.create_model(\n checkpoint['pad_len'], \n checkpoint['imu_dim'], \n checkpoint['tof_dim'], \n checkpoint['n_classes']\n ).to(self.device)\n \n \n \n model.load_state_dict({k.replace('_orig_mod.', ''):v for k, v in checkpoint['model_state_dict'].items()})\n model.eval()\n self.models.append(model)\n \n print(\" model, scaler, pads loaded – ready for evaluation\")\n \n return \n \n def predict(self, sequence: pl.DataFrame, demographics: pl.DataFrame, nanratio = 0): \n if demographics[0, \"handedness\"] == 0: \n sequence = self.preprocess_left_handed(sequence)\n \n df_seq = sequence.to_pandas()\n df_seq, _, _ = self.get_new_features(df_seq)\n \n with torch.no_grad():\n outputs = None\n outputs2 = None\n \n mat = self.preprocess_sequence(df_seq, self.feature_cols, None)\n pad = self.pad_sequences_torch([mat], maxlen=self.pad_len, padding='pre', truncating='pre')\n x = torch.FloatTensor(pad).float().to(self.device)\n\n if nanratio==1:\n x[:,:,7:] = 0.\n \n for model in self.models:\n model.eval()\n p, p2 = model(x)\n if outputs is None: \n outputs = p\n outputs2 = p2\n else: \n outputs += p\n outputs2 += p2\n \n outputs /= len(self.models)\n outputs2 /= len(self.models)\n\n return self.gesture_classes, (outputs.cpu().numpy(), outputs2.cpu().numpy()[:,:4])","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-09-02T05:28:09.101351Z","iopub.execute_input":"2025-09-02T05:28:09.10168Z","iopub.status.idle":"2025-09-02T05:28:09.261956Z","shell.execute_reply.started":"2025-09-02T05:28:09.10166Z","shell.execute_reply":"2025-09-02T05:28:09.26139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_zhou_inference_v26_1 = model_zhou_v26('/kaggle/input/cmi3v90', 42, './42')\nif VALIDATION:\n MMM = model_zhou_inference_v26_1\n \n MMM.TRAIN_FOLDS = [0, 1, 2, 3, 4, ]\n \n MMM.VALIDATION = VALIDATION\n guestures, pred_all_26_1, pred_imuonly_26_1, targets, idlist_26_1, pred_orientation_26_1, target_orientation_26_1 = MMM.train()\n\n val_score_imuonly = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_imuonly_26_1.argmax(1)]})\n )\n val_score_all = CompetitionMetric().calculate_hierarchical_f1(\n pd.DataFrame({'gesture': guestures[targets]}),\n pd.DataFrame({'gesture': guestures[pred_all_26_1.argmax(1)]})\n )\n\n print('score imuonly : ', val_score_imuonly)\n print('score allfeatures : ', val_score_all)\n \n MMM.VALIDATION = False","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:28:09.263293Z","iopub.execute_input":"2025-09-02T05:28:09.263976Z","iopub.status.idle":"2025-09-02T05:28:09.27008Z","shell.execute_reply.started":"2025-09-02T05:28:09.263948Z","shell.execute_reply":"2025-09-02T05:28:09.269362Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"imuonly_models = [\n model_zhou_inference_v14_1,\n model_zhou_inference_v14_2,\n model_zhou_inference_v14_3,\n\n model_zhou_inference_v17_1,\n \n model_zhou_inference_v20_1,\n model_zhou_inference_v21_1,\n\n model_zhou_inference_v24_1,\n]\n\nall_feature_models = [\n model_zhou_inference_v15_1,\n model_zhou_inference_v15_2,\n model_zhou_inference_v15_3,\n \n model_zhou_inference_v16_1,\n\n \n model_zhou_inference_v1_1,\n model_zhou_inference_v1_2,\n\n model_zhou_inference_v22_1,\n model_zhou_inference_v26_1,\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:28:11.783746Z","iopub.execute_input":"2025-09-02T05:28:11.784023Z","iopub.status.idle":"2025-09-02T05:28:11.792694Z","shell.execute_reply.started":"2025-09-02T05:28:11.784004Z","shell.execute_reply":"2025-09-02T05:28:11.792023Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PREDICTION","metadata":{}},{"cell_type":"code","source":"pad_lens_1branch = [np.load(imu_features_lstm_dir / f\"sequence_maxlen_fold{i}.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_1branch2 = [np.load(imu_features_lstm_dir2 / f\"sequence_maxlen_fold{i}.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_1branch_att = [np.load(imu_features_attention_dir / f\"sequence_maxlen_fold{i}.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_1branch_cnn = [np.load(imu_features_cnn_dir / f\"sequence_maxlen_fold{i}.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_1branch = pad_lens_1branch + pad_lens_1branch2 + pad_lens_1branch_att + \\\n pad_lens_1branch_cnn\n\npad_lens_1branch = [i + 5 for i in pad_lens_1branch]\n \npad_lens_3branch = [np.load(all_features_lstm_dir / f\"sequence_maxlen_fold{i}_3branch.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_3branch2 = [np.load(all_features_lstm_dir2 / f\"sequence_maxlen_fold{i}_3branch.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_3branch_att = [np.load(all_features_attention_dir / f\"sequence_maxlen_fold{i}_3branch.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_3branch_cnn = [np.load(all_features_cnn_dir / f\"sequence_maxlen_fold{i}_3branch.npy\", \n allow_pickle = True).item() for i in range(5)]\npad_lens_3branch = pad_lens_3branch + pad_lens_3branch2 + pad_lens_3branch_att + pad_lens_3branch_cnn\n\npad_lens_3branch = [i + 5 for i in pad_lens_3branch]\n\nfeature_cols_ensemble_32 = np.load(all_features_cnn_dir / \"feature_cols.npy\", allow_pickle=True).tolist()\ntof_cols_ensemble_32 = [c for c in feature_cols_ensemble_32 if c.startswith('tof_')]\nimu_cols_ensemble_32 = np.load(imu_features_cnn_dir / \"feature_cols_imu.npy\", allow_pickle=True).tolist()\ngesture_classes = np.load(all_features_cnn_dir / \"gesture_classes.npy\", allow_pickle=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:28:19.107873Z","iopub.execute_input":"2025-09-02T05:28:19.108689Z","iopub.status.idle":"2025-09-02T05:28:19.162711Z","shell.execute_reply.started":"2025-09-02T05:28:19.108653Z","shell.execute_reply":"2025-09-02T05:28:19.162184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dict_orientation2 = {\n 3: 'Seated Straight',\n 2: 'Seated Lean Non Dom - FACE DOWN',\n 1: 'Lie on Side - Non Dominant',\n 0: 'Lie on Back'\n }","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"runi = 0\n\npp0 = PostProcessor0()\npp1 = PostProcessor0()\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n global runi\n global pp0, pp1\n \n if runi==0:\n print('start loading models zhou')\n for model in all_feature_models:\n model.get_model()\n\n for model in imuonly_models:\n model.get_model()\n \n runi += 1\n\n # predict for the orientations\n pred_orientation_list = []\n\n \n\n sequence_copy = sequence.clone()\n demographics_copy = demographics.clone()\n\n df_seq = sequence.to_pandas()\n dem = demographics.to_pandas()\n \n df_seq1 = sequence.to_pandas()\n dem1 = demographics.to_pandas()\n # \n\n if demographics[0, \"handedness\"] == 0: \n sequence = preprocess_left_handed(sequence)\n \n thm_tof = sequence[:,[\"thm_\" + str(x) for x in range(1,6)] + \n [\"tof_\" + str(x) + \"_v\"+str(y) for x in range(1,6) for y in range(64)]]\n\n inputs = torch.tensor(prep_data(sequence, imu_model_cols, scaler_imu).astype(np.float32)).to(device2)\n\n p_imu_cnngru = predict_inner(inputs, models_imu_cnngru)\n p_imu_cnn = predict_inner(inputs, models_imu_cnn)\n p_imu_densegru = predict_inner(inputs, models_imu_densegru)\n p_imu_cnngruSE = predict_inner(inputs, models_imu_cnngruSE)\n p_imu_cnnSE = predict_inner(inputs, models_imu_cnnSE)\n inputs = inputs.to(device)\n p_imu_cnngru_resgru = predict_inner(inputs, models_imu_cnngru_resgru)\n p_imu_orientation = predict_inner(inputs, models_imu_orientation)\n p_imu_orientation = p_imu_orientation.detach().cpu().numpy()\n \n p_imu = 0.1*p_imu_cnngru + 0.1*p_imu_cnn + 0.05* p_imu_densegru + 0.1*p_imu_cnngruSE+0.1*p_imu_cnnSE\n \n if thm_tof.null_count().transpose().sum()[0,0] < (thm_tof.shape[1]*thm_tof.shape[0])/2:\n inputs = torch.tensor(prep_data(sequence, all_model_cols, scaler_all).astype(np.float32)).to(device2)\n\n p_all_cnngru = predict_inner(inputs, models_all_cnngru)\n p_all_cnn = predict_inner(inputs, models_all_cnn) \n p_all_densegru = predict_inner(inputs, models_all_densegru)\n p_all_cnngruSE = predict_inner(inputs, models_all_cnngruSE)\n p_all_cnnSE = predict_inner(inputs, models_all_cnnSE) \n inputs = inputs.to(device)\n p_all_cnngru_resgru = predict_inner(inputs, models_all_cnngru_resgru)\n p_all_orientation = predict_inner(inputs, models_all_orientation)\n p_all_orientation = p_all_orientation.detach().cpu().numpy() + p_imu_orientation\n \n p_all = 0.1*p_all_cnngru + 0.1*p_all_cnn + 0.05 * p_all_densegru + 0.1*p_all_cnngruSE + 0.1*p_all_cnnSE\n \n inputs = torch.tensor(prep_data(sequence, all_model_diff_cols, scaler_all_diff).astype(np.float32)).to(device2)\n p_all_cnngru_diff = predict_inner(inputs, models_all_cnngru_diff)\n p_all_cnn_diff = predict_inner(inputs, models_all_cnn_diff) \n p_all_densegru_diff = predict_inner(inputs, models_all_densegru_diff)\n p_all_cnngruSE_diff = predict_inner(inputs, models_all_cnngruSE_diff)\n p_all_cnnSE_diff = predict_inner(inputs, models_all_cnnSE_diff) \n \n p_all_diff = 0.1*p_all_cnngru_diff + 0.1*p_all_cnn_diff + 0.05 * p_all_densegru_diff + 0.1*p_all_cnngruSE_diff + 0.1*p_all_cnnSE_diff\n p_all = p_all + p_all_diff\n \n tof_inputs = torch.tensor(prep_tof(sequence, tof_mean, tof_std).astype(np.float32)).to(device2)\n tof_diff_inputs = torch.tensor(prep_tof_diff(sequence, tof_diff_mean, tof_diff_std).astype(np.float32)).to(device2)\n\n p_tof = predict_inner(tof_inputs, models_all_tof3dnet)\n p_tof_diff = predict_inner(tof_diff_inputs, models_all_tof3dnet_diff)\n p_tof_combined = p_tof+p_tof_diff\n \n final_prediction = (p_all * 0.3 + p_tof_combined * 0.2 + p_imu*0.4) * 0.95 + p_all_cnngru_resgru.to(device2)*0.05\n else:\n final_prediction = p_imu * 0.9 + p_imu_cnngru_resgru.to(device2) * 0.1 \n # \n \n preds_imu = []\n preds_all = []\n\n df_seq[ROT_COLS] = handle_quaternion_missing_values(df_seq[ROT_COLS].to_numpy())\n\n if dem.loc[0, \"handedness\"] == 0:\n left_handed = df_seq\n cols_to_transform = [\"acc_x\", \"acc_y\", \"acc_z\", \"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]\n left_handed_arr = left_handed[cols_to_transform].to_numpy()\n df_seq.loc[:, cols_to_transform] = mirror_data(left_handed_arr)\n \n # Condition from Devin's part\n if thm_tof.null_count().transpose().sum()[0,0] < (thm_tof.shape[1]*thm_tof.shape[0])/2: \n df_seq = df_seq.ffill().bfill().fillna(0)\n mat = add_features(df_seq)[feature_cols_ensemble_32]\n mat = mat.to_numpy()\n for i, m in enumerate(models_all_features_old):\n pad = pad_sequences([mat], maxlen=pad_lens_3branch[i], padding='pre', truncating='pre', dtype='float32')\n pad = torch.from_numpy(pad).to(device)\n if i == 0:\n preds_all = m(pad)\n else:\n preds_all += m(pad)\n \n preds_all_old = preds_all.detach().cpu().numpy()\n\n if dem1.loc[0, \"handedness\"] == 0:\n df_seq1 = preprocess_left_handed_pd(df_seq1, dem1)\n df_seq1[ROT_COLS] = handle_quaternion_missing_values(df_seq1[ROT_COLS].to_numpy())\n df_seq1 = df_seq1.ffill().bfill().fillna(0)\n mat = add_features(df_seq1)[feature_cols_ensemble_32]\n mat = mat.to_numpy()\n\n for i, m in enumerate(models_all_features):\n pad = pad_sequences([mat], maxlen=pad_lens_3branch[i], padding='pre', truncating='pre', dtype='float32')\n pad = torch.from_numpy(pad).to(device)\n if i == 0:\n preds_all = m(pad)\n else:\n preds_all += m(pad)\n \n preds_all = preds_all.detach().cpu().numpy()\n\n preds_all = preds_all+preds_all_old\n \n # zhou\n pred_final_list_zhou = []\n for model in all_feature_models:\n gesture_classes_zhou, prediction_proba_zhou = model.predict(sequence_copy, demographics_copy, 0.)\n\n if type(prediction_proba_zhou) is tuple:\n pred_orientation_list.append(prediction_proba_zhou[1])\n prediction_proba_zhou = prediction_proba_zhou[0]\n else:\n pred_final_list_zhou.append(prediction_proba_zhou)\n\n pred_final_zhou_all = np.mean(pred_final_list_zhou, 0)\n\n pred_final_list_zhou = []\n for model in imuonly_models:\n gesture_classes_zhou, prediction_proba_zhou = model.predict(sequence_copy, demographics_copy, 1.)\n\n if type(prediction_proba_zhou) is tuple:\n pred_orientation_list.append(prediction_proba_zhou[1])\n prediction_proba_zhou = prediction_proba_zhou[0]\n else:\n pred_final_list_zhou.append(prediction_proba_zhou)\n pred_final_zhou_imu = np.mean(pred_final_list_zhou, 0)\n pred_final_zhou = pred_final_zhou_all + 0.2 * pred_final_zhou_imu\n \n pred_orientation_list = np.mean(pred_orientation_list, 0)\n pred_orientation_list = pred_orientation_list*0.975 + p_all_orientation*0.025\n orientation = dict_orientation2.get(pred_orientation_list[0].argmax(-1), 'Seated Lean Non Dom - FACE DOWN')\n \n # BLEND\n blend_preds = 0.43*preds_all + 0.57*final_prediction.cpu().numpy()\n # final_prediction = p_all * 0.3 + p_tof_combined * 0.2 + p_imu*0.4\n \n blend_preds = blend_preds * 0.025 + pred_final_zhou * 0.975\n\n blend_preds = blend_preds[0]\n\n print(blend_preds.shape)\n \n try:\n print('preprocessing for allfeat')\n result = pp0.process_prediction(blend_preds, demographics_copy[0, \"subject\"], orientation)\n print('finish')\n return result\n except:\n idx = int(blend_preds.argmax(1))\n return str(gesture_classes[idx])\n\n else:\n df_seq = df_seq.ffill().bfill().fillna(0)\n mat = add_features(df_seq)[imu_cols_ensemble_32]\n mat = mat.to_numpy()\n for i, m in enumerate(models_imu_features):\n pad = pad_sequences([mat], maxlen=pad_lens_1branch[i], padding='pre', truncating='pre', dtype='float32')\n pad = torch.from_numpy(pad).to(device)\n if i == 0:\n preds_imu = m(pad)\n else:\n preds_imu += m(pad)\n \n preds_imu = preds_imu.detach().cpu().numpy()\n\n pred_final_list_zhou = []\n for model in imuonly_models:\n gesture_classes_zhou, prediction_proba_zhou = model.predict(sequence_copy, demographics_copy, 1.)\n if type(prediction_proba_zhou) is tuple:\n pred_orientation_list.append(prediction_proba_zhou[1])\n prediction_proba_zhou = prediction_proba_zhou[0]\n else:\n pred_final_list_zhou.append(prediction_proba_zhou) \n pred_final_zhou_imu = np.mean(pred_final_list_zhou, 0)\n \n pred_orientation_list = np.mean(pred_orientation_list, 0)\n pred_orientation_list = pred_orientation_list*0.975 + p_imu_orientation*0.025\n orientation = dict_orientation2.get(pred_orientation_list[0].argmax(-1), 'Seated Lean Non Dom - FACE DOWN')\n\n # BLEND\n blend_preds = 0.5*preds_imu + 0.5*final_prediction.cpu().numpy()\n blend_preds = 0.03*blend_preds + 0.97*pred_final_zhou_imu\n #blend_preds = 0.997*blend_preds + 0.003*pred_spec\n\n blend_preds = blend_preds[0]\n\n print(blend_preds.shape)\n \n try:\n print('preprocessing for imuonly')\n result = pp1.process_prediction(blend_preds, demographics_copy[0, \"subject\"], orientation)\n print('finish')\n return result\n except:\n idx = int(blend_preds.argmax(1))\n return str(gesture_classes[idx])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:30:58.963444Z","iopub.execute_input":"2025-09-02T05:30:58.963783Z","iopub.status.idle":"2025-09-02T05:30:58.991255Z","shell.execute_reply.started":"2025-09-02T05:30:58.96376Z","shell.execute_reply":"2025-09-02T05:30:58.990548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import kaggle_evaluation.cmi_inference_server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n inference_server.serve()\nelse:\n inference_server.run_local_gateway(\n data_paths=(\n \"/kaggle/input/cmi-detect-behavior/test_with_nulls.csv\",\n '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n #\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\",\n #\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\",\n )\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-02T05:31:00.975789Z","iopub.execute_input":"2025-09-02T05:31:00.976542Z","iopub.status.idle":"2025-09-02T05:31:28.383111Z","shell.execute_reply.started":"2025-09-02T05:31:00.976506Z","shell.execute_reply":"2025-09-02T05:31:28.382491Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}