{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12915524,"sourceType":"datasetVersion","datasetId":7656808}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"papermill":{"default_parameters":{},"duration":305.575685,"end_time":"2025-06-08T08:07:13.048918","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-06-08T08:02:07.473233","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q optuna==4.5.0 scikit-learn==1.5.2","metadata":{"trusted":true,"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Imports and configs","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nfrom sklearn.utils import shuffle\nfrom sklearn.base import clone\nfrom scipy.spatial.transform import Rotation as R\nfrom scipy.special import logit\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport polars as pl\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport joblib\nimport shutil\nimport optuna\nimport glob\nimport json\nimport gc\nimport os\n\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    train_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\"\n    train_metadata_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\"\n    \n    test_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\"\n    test_metadata_path = \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\"\n\n    data_path = \"/kaggle/input/cmi-2025-bfrb-prediction-pretrained-models/data\"\n    models_path = \"/kaggle/input/cmi-2025-bfrb-prediction-pretrained-models/models\"\n    \n    n_folds = 5\n    seed = 42\n\n    run_optuna = True\n    n_optuna_trials = 500","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data loading and preprocessing","metadata":{}},{"cell_type":"code","source":"num_to_label = {\n    0: \"Above ear - pull hair\",\n    1: \"Cheek - pinch skin\",\n    2: \"Eyebrow - pull hair\",\n    3: \"Eyelash - pull hair\",\n    4: \"Forehead - pull hairline\",\n    5: \"Forehead - scratch\",\n    6: \"Neck - pinch skin\",\n    7: \"Neck - scratch\",\n    8: \"Drink from bottle/cup\",\n    9: \"Feel around in tray and pull out an object\",\n    10: \"Glasses on/off\",\n    11: \"Pinch knee/leg skin\",\n    12: \"Pull air toward your face\",\n    13: \"Scratch knee/leg skin\",\n    14: \"Text on phone\",\n    15: \"Wave hello\",\n    16: \"Write name in air\",\n    17: \"Write name on leg\"\n}\n\nlabel_to_num = {v: k for k, v in num_to_label.items()}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class NNPreprocessorPT:\n    def __init__(self, train_path=None, data_path=None):\n        self.train_path = train_path\n        self.data_path = data_path\n        \n        self.init_feature_names()\n\n    def init_feature_names(self):\n        self.target_gestures = [\n            'Above ear - pull hair',\n            'Cheek - pinch skin',\n            'Eyebrow - pull hair',\n            'Eyelash - pull hair',\n            'Forehead - pull hairline',\n            'Forehead - scratch',\n            'Neck - pinch skin',\n            'Neck - scratch',\n        ]\n        self.non_target_gestures = [\n            'Write name on leg',\n            'Wave hello',\n            'Glasses on/off',\n            'Text on phone',\n            'Write name in air',\n            'Feel around in tray and pull out an object',\n            'Scratch knee/leg skin',\n            'Pull air toward your face',\n            'Drink from bottle/cup',\n            'Pinch knee/leg skin'\n        ]\n\n        self.imu_cols = [\n            'rot_x', 'rot_y', 'rot_z', 'rot_w', \n            'acc_mag', 'acc_mag_jerk', \n            'rot_angle', 'rot_angle_vel', \n            'linear_acc_x', 'linear_acc_y', 'linear_acc_z', 'linear_acc_mag', 'linear_acc_mag_jerk', \n            'angular_vel_x', 'angular_vel_y', 'angular_vel_z', 'angular_distance'\n        ]\n        self.imu_channel_keys = defaultdict(list)\n        self.imu_channel_keys[\"acc\"] = ['linear_acc_x', 'linear_acc_y', 'linear_acc_z']\n        self.imu_channel_keys[\"rot\"] = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\n        self.imu_channel_keys[\"other\"] = ['acc_mag', 'acc_mag_jerk', 'linear_acc_mag', 'linear_acc_mag_jerk', 'rot_angle', 'rot_angle_vel', 'angular_vel_x', 'angular_vel_y', 'angular_vel_z', 'angular_distance']\n\n        self.thm_cols = [f\"thm_{i}\" for i in range(1, 6)]\n        self.thm_channel_keys = {k: [f\"thm_{k}\"] for k in range(1, 6)}\n        \n        self.tof_cols = []\n        self.tof_channel_keys = defaultdict(list)\n        for i in range(1, 6):\n            self.tof_cols.extend([f\"tof_{i}_v{p}\" for p in range(64)])\n            self.tof_channel_keys[i].extend([f\"tof_{i}_v{p}\" for p in range(64)])\n            \n            for stat in ['mean', 'std', 'min', 'max']:\n                self.tof_cols.append(f'tof_{i}_{stat}')\n                self.tof_channel_keys[i].append(f'tof_{i}_{stat}')\n\n            for r in range(16):\n                for stat in ['mean', 'std', 'min', 'max']:\n                    self.tof_cols.append(f'tof_16_{i}_region_{r}_{stat}')\n                    self.tof_channel_keys[i].append(f'tof_16_{i}_region_{r}_{stat}')\n\n        self.imu_dim = len(self.imu_cols)\n        self.thm_dim = len(self.thm_cols)\n        self.tof_dim = len(self.tof_cols)\n        \n        self.global_imu_indices = {k: sorted([self.imu_cols.index(feat) for feat in feats]) for k, feats in self.imu_channel_keys.items()}\n        self.global_thm_indices = {k: sorted([self.thm_cols.index(key) for key in self.thm_channel_keys[k]]) for k in range(1, 6)}\n        self.global_tof_indices = {k: sorted([self.tof_cols.index(key) for key in self.tof_channel_keys[k]]) for k in range(1, 6)}\n        \n    def remove_gravity_from_acc(self, acc_data, rot_data):\n        if isinstance(acc_data, pd.DataFrame):\n            acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n        else:\n            acc_values = acc_data\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = acc_values.shape[0]\n        linear_accel = np.zeros_like(acc_values)\n        gravity_world = np.array([0, 0, 9.81])\n        for i in range(num_samples):\n            if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n                linear_accel[i, :] = acc_values[i, :] \n                continue\n            try:\n                rotation = R.from_quat(quat_values[i])\n                gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n                linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n            except ValueError:\n                linear_accel[i, :] = acc_values[i, :]\n        return linear_accel\n\n    def calculate_angular_velocity_from_quat(self, rot_data, time_delta=1/200):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = quat_values.shape[0]\n        angular_vel = np.zeros((num_samples, 3))\n        for i in range(num_samples - 1):\n            q_t = quat_values[i]\n            q_t_plus_dt = quat_values[i+1]\n            if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n            np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n                continue\n            try:\n                rot_t = R.from_quat(q_t)\n                rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n                delta_rot = rot_t.inv() * rot_t_plus_dt\n                angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n            except ValueError:\n                pass\n        return angular_vel\n\n    def calculate_angular_distance(self, rot_data):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = quat_values.shape[0]\n        angular_dist = np.zeros(num_samples)\n        for i in range(num_samples - 1):\n            q1 = quat_values[i]\n            q2 = quat_values[i+1]\n            if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n            np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n                angular_dist[i] = 0\n                continue\n            try:\n                r1 = R.from_quat(q1)\n                r2 = R.from_quat(q2)\n                relative_rotation = r1.inv() * r2\n                angle = np.linalg.norm(relative_rotation.as_rotvec())\n                angular_dist[i] = angle\n            except ValueError:\n                angular_dist[i] = 0\n                pass\n        return angular_dist\n        \n    def generate_features(self, df, is_test=False):\n        if not is_test:\n            df['gesture_int'] = df[\"gesture\"].map(label_to_num)\n            \n        self.targets = np.array([label_to_num[name] for name in self.target_gestures])\n        self.non_targets = np.array([label_to_num[name] for name in self.non_target_gestures])\n        \n        df['acc_mag'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n        df['rot_angle'] = 2 * np.arccos(df['rot_w'].clip(-1, 1))\n        df['acc_mag_jerk'] = df.groupby('sequence_id')['acc_mag'].diff().fillna(0)\n        df['rot_angle_vel'] = df.groupby('sequence_id')['rot_angle'].diff().fillna(0)\n            \n        linear_accel_list = []\n        for _, group in df.groupby('sequence_id'):\n            acc_data_group = group[['acc_x', 'acc_y', 'acc_z']]\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            linear_accel_group = self.remove_gravity_from_acc(acc_data_group, rot_data_group)\n            linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['linear_acc_x', 'linear_acc_y', 'linear_acc_z'], index=group.index))\n       \n        df_linear_accel = pd.concat(linear_accel_list)\n        df = pd.concat([df, df_linear_accel], axis=1)\n        del linear_accel_list, df_linear_accel\n        gc.collect()\n       \n        df['linear_acc_mag'] = np.sqrt(df['linear_acc_x']**2 + df['linear_acc_y']**2 + df['linear_acc_z']**2)\n        df['linear_acc_mag_jerk'] = df.groupby('sequence_id')['linear_acc_mag'].diff().fillna(0)\n    \n        angular_vel_list = []\n        for _, group in df.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_vel_group = self.calculate_angular_velocity_from_quat(rot_data_group)\n            angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['angular_vel_x', 'angular_vel_y', 'angular_vel_z'], index=group.index))\n       \n        df_angular_vel = pd.concat(angular_vel_list)\n        df = pd.concat([df, df_angular_vel], axis=1)\n        del angular_vel_list, df_angular_vel\n        gc.collect()\n        \n        angular_distance_list = []\n        for _, group in df.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_dist_group = self.calculate_angular_distance(rot_data_group)\n            angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['angular_distance'], index=group.index))\n        \n        \n        df_angular_distance = pd.concat(angular_distance_list)\n        df = pd.concat([df, df_angular_distance], axis=1)\n        del angular_distance_list, df_angular_distance\n        gc.collect()\n\n        new_columns = {}\n        for i in range(1, 6):\n            pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n            tof_data = df[pixel_cols].replace(-1, np.nan)\n            new_columns.update({\n                f'tof_{i}_mean': tof_data.mean(axis=1),\n                f'tof_{i}_std': tof_data.std(axis=1),\n                f'tof_{i}_min': tof_data.min(axis=1),\n                f'tof_{i}_max': tof_data.max(axis=1)\n            })\n            region_size = 64 // 16\n            for r in range(16):\n                region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n                new_columns.update({\n                    f'tof_16_{i}_region_{r}_mean': region_data.mean(axis=1),\n                    f'tof_16_{i}_region_{r}_std': region_data.std(axis=1),\n                    f'tof_16_{i}_region_{r}_min': region_data.min(axis=1),\n                    f'tof_16_{i}_region_{r}_max': region_data.max(axis=1)\n                })\n\n        return pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n\n    def scale(self, data_unscaled, is_test=False, scaler_path=None, scaler_prefix=\"\"):\n        if is_test:\n            scaler = joblib.load(scaler_path)\n        else:\n            scaler = StandardScaler()\n            scaler = scaler.fit(np.concatenate(data_unscaled, axis=0))\n            joblib.dump(scaler, f\"{self.data_path}/{scaler_prefix}_scaler.pkl\")\n            \n        return [scaler.transform(x) for x in data_unscaled]\n        \n\n    def pad(self, data_scaled, cols, pad_len):\n        pad_data = np.zeros((len(data_scaled), pad_len, len(cols)), dtype='float32')\n        for i, seq in enumerate(data_scaled):\n            seq_len = min(len(seq), pad_len)\n            pad_data[i, :seq_len] = seq[:seq_len]\n        return pad_data\n\n    def process_train(self):\n        if os.path.exists(f\"{self.data_path}/X_imu.npy\"):\n            X_imu = np.load(f\"{self.data_path}/X_imu.npy\", mmap_mode='r')\n            X_thm = np.load(f\"{self.data_path}/X_thm.npy\", mmap_mode='r')\n            X_tof = np.load(f\"{self.data_path}/X_tof.npy\", mmap_mode='r')\n            targets_int = np.load(f\"{self.data_path}/targets_int.npy\", mmap_mode='r')\n            targets_ohe = np.load(f\"{self.data_path}/targets_ohe.npy\", mmap_mode='r')\n            groups = np.load(f\"{self.data_path}/groups.npy\", mmap_mode='r')\n            pad_len = joblib.load(f\"{self.data_path}/pad_len.pkl\")\n            \n            return X_imu, X_thm, X_tof, targets_int, targets_ohe, groups, pad_len\n        \n        os.makedirs(self.data_path, exist_ok=True)\n        \n        df = pd.read_csv(self.train_path)\n        df = self.generate_features(df, is_test=False)\n        \n        targets_int, lens, groups = [], [], []\n        imu_unscaled, thm_unscaled, tof_unscaled = [], [], []\n        for i in range(2):\n            for _, seq_df in df.groupby('sequence_id') :\n                imu_data = seq_df[self.imu_cols]\n                thm_data = seq_df[self.thm_cols]\n                tof_data = seq_df[self.tof_cols]\n                \n                imu_data = imu_data.ffill().bfill().fillna(0)\n                imu_unscaled.append(imu_data.values.astype('float32'))\n                if i == 0:\n                    thm_data = thm_data.ffill().bfill().fillna(0)\n                    thm_unscaled.append(thm_data.values.astype('float32'))\n\n                    tof_data = tof_data.ffill().bfill().fillna(0)\n                    tof_unscaled.append(tof_data.values.astype('float32'))\n                else:\n                    thm_unscaled.append(np.zeros((len(thm_data), len(self.thm_cols))))\n                    tof_unscaled.append(np.zeros((len(tof_data), len(self.tof_cols))))\n                \n                targets_int.append(seq_df['gesture_int'].iloc[0])\n                groups.append(seq_df['subject'].iloc[0])\n                lens.append(len(imu_data))\n            \n        pad_len = int(np.percentile(lens, 99))\n\n        imu_scaled = self.scale(imu_unscaled, False, None, \"imu\")\n        thm_scaled = self.scale(thm_unscaled, False, None, \"thm\")\n        tof_scaled = self.scale(tof_unscaled, False, None, \"tof\")\n        \n        X_imu = self.pad(imu_scaled, self.imu_cols, pad_len)\n        X_thm = self.pad(thm_scaled, self.thm_cols, pad_len)\n        X_tof = self.pad(tof_scaled, self.tof_cols, pad_len)\n\n        targets_ohe = F.one_hot(torch.from_numpy(np.array(targets_int)).long(), num_classes=18).float().numpy()\n        groups = np.array(groups)\n        targets_int = np.array(targets_int)\n        targets_ohe = np.array(targets_ohe)\n        \n        np.save(f\"{self.data_path}/X_imu.npy\", X_imu)\n        np.save(f\"{self.data_path}/X_thm.npy\", X_thm)\n        np.save(f\"{self.data_path}/X_tof.npy\", X_tof)\n        np.save(f\"{self.data_path}/targets_int.npy\", targets_int)\n        np.save(f\"{self.data_path}/targets_ohe.npy\", targets_ohe)\n        np.save(f\"{self.data_path}/groups.npy\", groups)\n        joblib.dump(pad_len, f\"{self.data_path}/pad_len.pkl\")\n        \n        return X_imu, X_thm, X_tof, targets_int, targets_ohe, groups, pad_len\n    \n\n    def process_test(self, sequence, pad_len, scaler_paths):\n        df = sequence.to_pandas()\n        df = self.generate_features(df, is_test=True)\n        \n        imu_unscaled, thm_unscaled, tof_unscaled = [], [], []\n        for _, seq_df in df.groupby('sequence_id'):\n            imu_data = seq_df[self.imu_cols]\n            imu_data = imu_data.ffill().bfill().fillna(0)\n            imu_unscaled.append(imu_data.values.astype('float32'))\n            \n            thm_data = seq_df[self.thm_cols]\n            thm_data = thm_data.ffill().bfill().fillna(0)\n            thm_unscaled.append(thm_data.values.astype('float32'))\n            \n            tof_data = seq_df[self.tof_cols]\n            tof_data = tof_data.ffill().bfill().fillna(0)\n            tof_unscaled.append(tof_data.values.astype('float32'))\n            \n        imu_scaled = self.scale(imu_unscaled, True, scaler_paths[\"imu\"])\n        thm_scaled = self.scale(thm_unscaled, True, scaler_paths[\"thm\"])\n        tof_scaled = self.scale(tof_unscaled, True, scaler_paths[\"tof\"])\n        \n        X_imu = self.pad(imu_scaled, self.imu_cols, pad_len)\n        X_thm = self.pad(thm_scaled, self.thm_cols, pad_len)\n        X_tof = self.pad(tof_scaled, self.tof_cols, pad_len)\n        \n        return X_imu, X_thm, X_tof\n\n\nclass NNPreprocessorTF:\n    def __init__(self, train_path=None, data_path=None):\n        self.train_path = train_path\n        self.data_path = data_path\n        \n        self.init_feature_names()\n        \n    def init_feature_names(self):\n        self.imu_cols = [\n            'acc_x', 'acc_y', 'acc_z', \n            'rot_x', 'rot_y', 'rot_z', 'rot_w', \n            'acc_mag', 'acc_mag_jerk', \n            'rot_angle', 'rot_angle_vel', \n            'linear_acc_x', 'linear_acc_y', 'linear_acc_z', 'linear_acc_mag', 'linear_acc_mag_jerk', \n            'angular_vel_x', 'angular_vel_y', 'angular_vel_z', 'angular_distance'\n        ]\n        \n        base_thm_cols = [f'thm_{i}' for i in range(1, 6)]\n        self.advanced_thm_cols = [\n            'thm_gradient_1_2',   \n        ]\n        \n        self.thm_cols = base_thm_cols + self.advanced_thm_cols\n\n        base_tof_cols = []\n        for i in range(1, 6):\n            base_tof_cols.extend([f\"tof_{i}_v{p}\" for p in range(64)])\n            \n            for stat in ['mean', 'std', 'min', 'max']:\n                base_tof_cols.append(f'tof_{i}_{stat}')\n                \n            for r in range(16):\n                for stat in ['mean', 'std', 'min', 'max']:\n                    base_tof_cols.append(f'tof_16_{i}_region_{r}_{stat}')\n        \n        self.advanced_tof_cols = [\n            'tof_3_center_mass_x',\n            'tof_5_spatial_gradient', \n            'tof_4_center_mass_x', \n        ]\n        \n        self.tof_cols = base_tof_cols + self.advanced_tof_cols\n        \n        self.feature_cols = self.imu_cols + self.thm_cols + self.tof_cols\n        \n        self.imu_dim = len(self.imu_cols)\n        self.thm_dim = len(self.thm_cols)\n        self.tof_dim = len(self.tof_cols)\n        \n    def remove_gravity_from_acc(self, acc_data, rot_data):\n        if isinstance(acc_data, pd.DataFrame):\n            acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n        else:\n            acc_values = acc_data\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = acc_values.shape[0]\n        linear_accel = np.zeros_like(acc_values)\n        gravity_world = np.array([0, 0, 9.81])\n        for i in range(num_samples):\n            if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n                linear_accel[i, :] = acc_values[i, :] \n                continue\n            try:\n                rotation = R.from_quat(quat_values[i])\n                gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n                linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n            except ValueError:\n                linear_accel[i, :] = acc_values[i, :]\n        return linear_accel\n\n    def calculate_angular_velocity_from_quat(self, rot_data, time_delta=1/200):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = quat_values.shape[0]\n        angular_vel = np.zeros((num_samples, 3))\n        for i in range(num_samples - 1):\n            q_t = quat_values[i]\n            q_t_plus_dt = quat_values[i+1]\n            if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n            np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n                continue\n            try:\n                rot_t = R.from_quat(q_t)\n                rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n                delta_rot = rot_t.inv() * rot_t_plus_dt\n                angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n            except ValueError:\n                pass\n        return angular_vel\n\n    def calculate_angular_distance(self, rot_data):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n        num_samples = quat_values.shape[0]\n        angular_dist = np.zeros(num_samples)\n        for i in range(num_samples - 1):\n            q1 = quat_values[i]\n            q2 = quat_values[i+1]\n            if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n            np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n                angular_dist[i] = 0\n                continue\n            try:\n                r1 = R.from_quat(q1)\n                r2 = R.from_quat(q2)\n                relative_rotation = r1.inv() * r2\n                angle = np.linalg.norm(relative_rotation.as_rotvec())\n                angular_dist[i] = angle\n            except ValueError:\n                angular_dist[i] = 0\n                pass\n        return angular_dist\n\n    def calculate_thermal_features(self, df):\n        thm_gradient_1_2 = np.abs(df['thm_1'].fillna(0) - df['thm_2'].fillna(0))\n        thm_gradient_2_3 = np.abs(df['thm_2'].fillna(0) - df['thm_3'].fillna(0))\n        thm_gradient_3_4 = np.abs(df['thm_3'].fillna(0) - df['thm_4'].fillna(0))\n        thm_gradient_4_5 = np.abs(df['thm_4'].fillna(0) - df['thm_5'].fillna(0))\n        \n        thm_cols = ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']\n        thm_change_rate_mean = df[thm_cols].diff().abs().mean(axis=1).fillna(0)\n        \n        thm_mean = df[thm_cols].mean(axis=1)\n        thm_std = df[thm_cols].std(axis=1)\n        thm_distribution_pattern = (thm_std / (thm_mean + 1e-6)).fillna(0)\n        \n        return (thm_gradient_1_2, thm_gradient_2_3, thm_gradient_3_4, thm_gradient_4_5,\n                thm_change_rate_mean, thm_distribution_pattern)\n\n    def calculate_tof_spatial_features(self, df):\n        spatial_features = {}\n        \n        for sensor_id in range(1, 6):\n            tof_pixels = df[[f\"tof_{sensor_id}_v{p}\" for p in range(64)]].fillna(0)\n            \n            spatial_gradients = []\n            center_mass_x_list = []\n            center_mass_y_list = []\n            \n            for idx, row in tof_pixels.iterrows():\n                grid = row.values.reshape(8, 8)\n                \n                grad_x = np.gradient(grid, axis=1)\n                grad_y = np.gradient(grid, axis=0)\n                gradient_mag = np.sqrt(grad_x**2 + grad_y**2).mean()\n                spatial_gradients.append(gradient_mag)\n                \n                if np.sum(grid) > 0:\n                    y_coords, x_coords = np.mgrid[0:8, 0:8]\n                    center_x = np.sum(x_coords * grid) / np.sum(grid)\n                    center_y = np.sum(y_coords * grid) / np.sum(grid)\n                else:\n                    center_x, center_y = 4.0, 4.0\n                    \n                center_mass_x_list.append(center_x)\n                center_mass_y_list.append(center_y)\n            \n            spatial_features[f'tof_{sensor_id}_spatial_gradient'] = pd.Series(spatial_gradients, index=df.index)\n            spatial_features[f'tof_{sensor_id}_center_mass_x'] = pd.Series(center_mass_x_list, index=df.index)\n            spatial_features[f'tof_{sensor_id}_center_mass_y'] = pd.Series(center_mass_y_list, index=df.index)\n        \n        return spatial_features\n    \n    def generate_features(self, df, is_test=False):     \n        if not is_test:\n            df['gesture_int'] = df['gesture'].map(label_to_num)\n\n        df['acc_mag'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n        df['rot_angle'] = 2 * np.arccos(df['rot_w'].clip(-1, 1))\n        df['acc_mag_jerk'] = df.groupby('sequence_id')['acc_mag'].diff().fillna(0)\n        df['rot_angle_vel'] = df.groupby('sequence_id')['rot_angle'].diff().fillna(0)\n\n        linear_accel_list = []\n        for _, group in df.groupby('sequence_id'):\n            acc_data_group = group[['acc_x', 'acc_y', 'acc_z']]\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            linear_accel_group = self.remove_gravity_from_acc(acc_data_group, rot_data_group)\n            linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['linear_acc_x', 'linear_acc_y', 'linear_acc_z'], index=group.index))\n\n        df_linear_accel = pd.concat(linear_accel_list)\n        df = pd.concat([df, df_linear_accel], axis=1)\n        del df_linear_accel, linear_accel_list\n        gc.collect()\n        \n        df['linear_acc_mag'] = np.sqrt(df['linear_acc_x']**2 + df['linear_acc_y']**2 + df['linear_acc_z']**2)\n        df['linear_acc_mag_jerk'] = df.groupby('sequence_id')['linear_acc_mag'].diff().fillna(0)\n\n        angular_vel_list = []\n        for _, group in df.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_vel_group = self.calculate_angular_velocity_from_quat(rot_data_group)\n            angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['angular_vel_x', 'angular_vel_y', 'angular_vel_z'], index=group.index))\n        \n        df_angular_vel = pd.concat(angular_vel_list)\n        df = pd.concat([df, df_angular_vel], axis=1)\n        del angular_vel_list, df_angular_vel\n        gc.collect()\n\n        angular_distance_list = []\n        for _, group in df.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_dist_group = self.calculate_angular_distance(rot_data_group)\n            angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['angular_distance'], index=group.index))\n            \n        df_angular_distance = pd.concat(angular_distance_list)\n        df = pd.concat([df, df_angular_distance], axis=1)\n        del angular_distance_list, df_angular_distance\n        gc.collect()\n\n        new_columns = {}\n        for i in range(1, 6):\n            pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n            tof_data = df[pixel_cols].replace(-1, np.nan)\n            new_columns.update({\n                f'tof_{i}_mean': tof_data.mean(axis=1),\n                f'tof_{i}_std': tof_data.std(axis=1),\n                f'tof_{i}_min': tof_data.min(axis=1),\n                f'tof_{i}_max': tof_data.max(axis=1)\n            })\n\n            region_size = 64 // 16\n            for r in range(16):\n                region_data = tof_data.iloc[:, r*region_size : (r+1)*region_size]\n                new_columns.update({\n                    f'tof_16_{i}_region_{r}_mean': region_data.mean(axis=1),\n                    f'tof_16_{i}_region_{r}_std': region_data.std(axis=1),\n                    f'tof_16_{i}_region_{r}_min': region_data.min(axis=1),\n                    f'tof_16_{i}_region_{r}_max': region_data.max(axis=1)\n                })\n                del region_data\n\n            del tof_data\n        \n        df = pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n        del new_columns\n        gc.collect()\n\n        advanced_features = {}\n        \n        for seq_id, group in df.groupby('sequence_id'):                        \n            (thm_gradient_1_2, thm_gradient_2_3, thm_gradient_3_4, thm_gradient_4_5,\n             thm_change_rate_mean, thm_distribution_pattern) = self.calculate_thermal_features(group)\n            advanced_features.update({\n                ('thm_gradient_1_2', seq_id): thm_gradient_1_2,\n                ('thm_gradient_2_3', seq_id): thm_gradient_2_3,\n                ('thm_gradient_3_4', seq_id): thm_gradient_3_4,\n                ('thm_gradient_4_5', seq_id): thm_gradient_4_5,\n                ('thm_change_rate_mean', seq_id): thm_change_rate_mean,\n                ('thm_distribution_pattern', seq_id): thm_distribution_pattern\n            })\n            \n            tof_spatial_features = self.calculate_tof_spatial_features(group)\n            for feature_name, feature_data in tof_spatial_features.items():\n                advanced_features[(feature_name, seq_id)] = feature_data\n        \n        thm_feature_names = self.advanced_thm_cols\n        tof_feature_names = self.advanced_tof_cols\n        \n        for feature_name in thm_feature_names:\n            feature_data = []\n            for seq_id in df['sequence_id'].unique():\n                feature_data.append(advanced_features[(feature_name, seq_id)])\n            df[feature_name] = pd.concat(feature_data)\n        \n        for feature_name in tof_feature_names:\n            feature_data = []\n            for seq_id in df['sequence_id'].unique():\n                feature_data.append(advanced_features[(feature_name, seq_id)])\n            df[feature_name] = pd.concat(feature_data)\n        \n        return df\n    \n    def scale(self, data_unscaled, is_test=False, scaler_path=None):\n        if is_test:\n            scaler = joblib.load(scaler_path)\n        else:\n            scaler = StandardScaler()\n            scaler = scaler.fit(np.concatenate(data_unscaled, axis=0))\n            joblib.dump(scaler, f\"{self.data_path}/scaler.pkl\")\n        \n        return [scaler.transform(x) for x in data_unscaled]\n\n    def pad(self, data_scaled, cols, pad_len):\n        pad_data = np.zeros((len(data_scaled), pad_len, len(cols)), dtype='float32')\n        for i, seq in enumerate(data_scaled):\n            seq_len = min(len(seq), pad_len)\n            pad_data[i, :seq_len] = seq[:seq_len]\n        return pad_data  \n    \n    def process_train(self):\n        if os.path.exists(f\"{self.data_path}/X_imu.npy\"):\n            X_imu = np.load(f\"{self.data_path}/X_imu.npy\", mmap_mode='r')\n            X_thm = np.load(f\"{self.data_path}/X_thm.npy\", mmap_mode='r')\n            X_tof = np.load(f\"{self.data_path}/X_tof.npy\", mmap_mode='r')\n            targets_int = np.load(f\"{self.data_path}/targets_int.npy\", mmap_mode='r')\n            targets_ohe = np.load(f\"{self.data_path}/targets_ohe.npy\", mmap_mode='r')\n            groups = np.load(f\"{self.data_path}/groups.npy\", mmap_mode='r')\n            pad_len = joblib.load(f\"{self.data_path}/pad_len.pkl\")\n            \n            return X_imu, X_thm, X_tof, targets_int, targets_ohe, groups, pad_len\n\n        os.makedirs(self.data_path, exist_ok=True)\n\n        df = pd.read_csv(CFG.train_path)\n        df = self.generate_features(df, is_test=False)\n\n        targets_int, lens, groups = [], [], []\n        imu_unscaled, thm_unscaled, tof_unscaled = [], [], []\n        for i in range(2):\n            for _, seq_df in df.groupby('sequence_id'):\n                imu_data = seq_df[self.imu_cols]\n                thm_data = seq_df[self.thm_cols]\n                tof_data = seq_df[self.tof_cols]\n\n                imu_data = imu_data.ffill().bfill().fillna(0)\n                imu_unscaled.append(imu_data.values.astype('float32'))\n                if i == 0:\n                    thm_data = thm_data.ffill().bfill().fillna(0)\n                    thm_unscaled.append(thm_data.values.astype('float32'))\n\n                    tof_data = tof_data.ffill().bfill().fillna(0)\n                    tof_unscaled.append(tof_data.values.astype('float32'))\n                else:\n                    thm_unscaled.append(np.zeros((len(thm_data), len(self.thm_cols))))\n                    tof_unscaled.append(np.zeros((len(tof_data), len(self.tof_cols))))\n                \n                targets_int.append(seq_df['gesture_int'].iloc[0])\n                groups.append(seq_df['subject'].iloc[0])\n                lens.append(len(imu_data))\n\n        pad_len = int(np.percentile(lens, 99))\n        X_unscaled = [np.concatenate([imu, thm, tof], axis=1) for imu, thm, tof in zip(imu_unscaled, thm_unscaled, tof_unscaled)]\n        X_scaled = self.scale(X_unscaled, is_test=False)\n        X_padded = self.pad(X_scaled, self.imu_cols+self.thm_cols+self.tof_cols, pad_len)\n        \n        X_imu = X_padded[..., :self.imu_dim]\n        X_thm = X_padded[..., self.imu_dim:self.imu_dim+self.thm_dim]\n        X_tof = X_padded[..., self.imu_dim+self.thm_dim:self.imu_dim+self.thm_dim+self.tof_dim]\n\n        targets_ohe = F.one_hot(torch.from_numpy(np.array(targets_int)).long(), num_classes=18).float().numpy()\n        groups = np.array(groups)\n        targets_int = np.array(targets_int)\n        targets_ohe = np.array(targets_ohe)\n\n        np.save(f\"{self.data_path}/X_imu.npy\", X_imu)\n        np.save(f\"{self.data_path}/X_thm.npy\", X_thm)\n        np.save(f\"{self.data_path}/X_tof.npy\", X_tof)\n        np.save(f\"{self.data_path}/targets_int.npy\", targets_int)\n        np.save(f\"{self.data_path}/targets_ohe.npy\", targets_ohe)\n        np.save(f\"{self.data_path}/groups.npy\", groups)\n        joblib.dump(pad_len, f\"{self.data_path}/pad_len.pkl\")\n        \n        return X_imu, X_thm, X_tof, targets_int, targets_ohe, groups, pad_len\n    \n    def process_test(self, sequence, pad_len, scaler_path):\n        df = sequence.to_pandas()\n        df = self.generate_features(df, is_test=True)\n        \n        imu_unscaled, thm_unscaled, tof_unscaled = [], [], []\n        for _, seq_df in df.groupby('sequence_id'):\n            imu_data = seq_df[self.imu_cols]\n            imu_data = imu_data.ffill().bfill().fillna(0)\n            imu_unscaled.append(imu_data.values.astype('float32'))\n\n            thm_data = seq_df[self.thm_cols]\n            thm_data = thm_data.ffill().bfill().fillna(0)\n            thm_unscaled.append(thm_data.values.astype('float32'))\n            \n            tof_data = seq_df[self.tof_cols]\n            tof_data = tof_data.ffill().bfill().fillna(0)\n            tof_unscaled.append(tof_data.values.astype('float32'))\n            \n        X_unscaled = [np.concatenate([imu, thm, tof], axis=1) for imu, thm, tof in zip(imu_unscaled, thm_unscaled, tof_unscaled)]\n        X_scaled = self.scale(X_unscaled, is_test=True, scaler_path=scaler_path)\n        X_padded = self.pad(X_scaled, self.imu_cols+self.thm_cols+self.tof_cols, pad_len)\n\n        X_imu = X_padded[..., :self.imu_dim]\n        X_thm = X_padded[..., self.imu_dim:self.imu_dim+self.thm_dim]\n        X_tof = X_padded[..., self.imu_dim+self.thm_dim:self.imu_dim+self.thm_dim+self.tof_dim]\n        \n        return X_imu, X_thm, X_tof","metadata":{"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GBMPreprocessor:\n    def remove_gravity_from_acc(self, acc_data, rot_data):\n        if isinstance(acc_data, pd.DataFrame):\n            acc_values = acc_data[['acc_x', 'acc_y', 'acc_z']].values\n        else:\n            acc_values = acc_data\n\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n\n        num_samples = acc_values.shape[0]\n        linear_accel = np.zeros_like(acc_values)\n        \n        gravity_world = np.array([0, 0, 9.81])\n\n        for i in range(num_samples):\n            if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n                linear_accel[i, :] = acc_values[i, :] \n                continue\n\n            try:\n                rotation = R.from_quat(quat_values[i])\n                gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n                linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n            except ValueError:\n                linear_accel[i, :] = acc_values[i, :]\n                \n        return linear_accel\n\n    def calculate_angular_velocity_from_quat(self, rot_data, time_delta=1/200):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n\n        num_samples = quat_values.shape[0]\n        angular_vel = np.zeros((num_samples, 3))\n\n        for i in range(num_samples - 1):\n            q_t = quat_values[i]\n            q_t_plus_dt = quat_values[i+1]\n\n            if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n            np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n                continue\n\n            try:\n                rot_t = R.from_quat(q_t)\n                rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n\n                delta_rot = rot_t.inv() * rot_t_plus_dt\n                \n                angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n            except ValueError:\n                pass\n                \n        return angular_vel\n\n    def calculate_angular_distance(self, rot_data):\n        if isinstance(rot_data, pd.DataFrame):\n            quat_values = rot_data[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n        else:\n            quat_values = rot_data\n\n        num_samples = quat_values.shape[0]\n        angular_dist = np.zeros(num_samples)\n\n        for i in range(num_samples - 1):\n            q1 = quat_values[i]\n            q2 = quat_values[i+1]\n\n            if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n            np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n                angular_dist[i] = 0\n                continue\n            try:\n                r1 = R.from_quat(q1)\n                r2 = R.from_quat(q2)\n\n                relative_rotation = r1.inv() * r2\n                \n                angle = np.linalg.norm(relative_rotation.as_rotvec())\n                angular_dist[i] = angle\n            except ValueError:\n                angular_dist[i] = 0\n                pass\n                \n        return angular_dist\n\n    def add_linear_acc_to_dataset(self, dataset):\n        dataset = dataset.to_pandas()\n        \n        linear_accel_list = []\n        for _, group in dataset.groupby('sequence_id'):\n            acc_data_group = group[['acc_x', 'acc_y', 'acc_z']]\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            linear_accel_group = self.remove_gravity_from_acc(acc_data_group, rot_data_group)\n            linear_accel_list.append(pd.DataFrame(linear_accel_group, columns=['linear_acc_x', 'linear_acc_y', 'linear_acc_z'], index=group.index))    \n        \n        linear_accel_df = pd.concat(linear_accel_list)\n        dataset = pd.concat([dataset, linear_accel_df], axis=1)\n        \n        return  pl.from_pandas(dataset)\n        \n    def add_angular_vel_to_dataset(self, dataset):\n        dataset = dataset.to_pandas()\n\n        angular_vel_list = []\n        for _, group in dataset.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_vel_group = self.calculate_angular_velocity_from_quat(rot_data_group)\n            angular_vel_list.append(pd.DataFrame(angular_vel_group, columns=['angular_vel_x', 'angular_vel_y', 'angular_vel_z'], index=group.index))\n        \n        angular_vel_df = pd.concat(angular_vel_list)\n        dataset = pd.concat([dataset, angular_vel_df], axis=1)\n        \n        return  pl.from_pandas(dataset)\n\n    def add_angular_distance_to_dataset(self, dataset):\n        dataset = dataset.to_pandas()\n        \n        angular_distance_list = []\n        for _, group in dataset.groupby('sequence_id'):\n            rot_data_group = group[['rot_x', 'rot_y', 'rot_z', 'rot_w']]\n            angular_dist_group = self.calculate_angular_distance(rot_data_group)\n            angular_distance_list.append(pd.DataFrame(angular_dist_group, columns=['angular_distance'], index=group.index))\n        \n        angular_distance_df = pd.concat(angular_distance_list)\n        dataset = pd.concat([dataset, angular_distance_df], axis=1)\n        \n        return pl.from_pandas(dataset)\n\n    def get_imu_data(self, dataset):\n        imu_cols = [col for col in dataset.columns if 'acc_' in col or 'rot_' in col]\n        \n        dataset = self.add_linear_acc_to_dataset(dataset)\n        dataset = self.add_angular_vel_to_dataset(dataset)\n        dataset = self.add_angular_distance_to_dataset(dataset)\n        \n        imu_cols.extend([\n            'linear_acc_x', 'linear_acc_y', 'linear_acc_z', \n            'angular_vel_x', 'angular_vel_y', 'angular_vel_z', \n            'angular_distance'\n        ])\n        \n        imu_aggs = []    \n        for col in imu_cols:        \n            imu_aggs.extend([\n                pl.mean(col).alias(f'{col}_mean'),\n                pl.std(col).alias(f'{col}_std'),\n                pl.max(col).alias(f'{col}_max'),\n                pl.min(col).alias(f'{col}_min'),\n                pl.quantile(col, 0.1).alias(f'{col}_q10'),\n                pl.quantile(col, 0.9).alias(f'{col}_q90'),\n                pl.median(col).alias(f'{col}_median'),\n                (pl.col(col).diff().fill_null(0)).std().alias(f'{col}_diff_std'),\n                (pl.col(col).diff().fill_null(0)).max().alias(f'{col}_diff_max'),\n                (pl.col(col).diff().fill_null(0)).min().alias(f'{col}_diff_min'),\n                (pl.col(col).diff(2).fill_null(0)).mean().alias(f'{col}_diff2_mean'),\n                (pl.col(col).skew()).alias(f'{col}_skew'),\n                (pl.col(col).kurtosis()).alias(f'{col}_kurtosis'),\n                (pl.col(col) - pl.col(col).shift(1)).abs().mean().alias(f'{col}_mad'),\n                (pl.col(col).max() - pl.col(col).min()).alias(f'{col}_range'),\n                (pl.col(col).quantile(0.75) - pl.col(col).quantile(0.25)).alias(f'{col}_iqr'),\n                pl.col(col).rolling_mean(window_size=5).std().alias(f'{col}_rolling5_std'),\n                pl.col(col).rolling_mean(window_size=10).std().alias(f'{col}_rolling10_std'),\n                pl.col(col).rolling_max(window_size=5).mean().alias(f'{col}_rolling5_max_mean'),\n                pl.col(col).rolling_min(window_size=5).mean().alias(f'{col}_rolling5_min_mean'),\n                (pl.col(col) > 0).sum().alias(f'{col}_positive_count'),\n                (pl.col(col).abs() > pl.col(col).abs().mean()).sum().alias(f'{col}_above_mean_count'),\n                ((pl.col(col) > 0) & (pl.col(col).shift(1) <= 0) | (pl.col(col) <= 0) & (pl.col(col).shift(1) > 0)).sum().alias(f'{col}_zero_crossings'),\n                (pl.col(col).rolling_max(window_size=3) == pl.col(col)).sum().alias(f'{col}_local_maxima_count'),\n                (pl.col(col).rolling_min(window_size=3) == pl.col(col)).sum().alias(f'{col}_local_minima_count'),\n                (pl.col(col).abs() > pl.col(col).abs().std()).sum().alias(f'{col}_above_std_count'),\n                (pl.col(col) - pl.col(col).median()).abs().median().alias(f'{col}_mad_median'),\n                pl.col(col).quantile(0.95).alias(f'{col}_q95'),\n                pl.col(col).quantile(0.05).alias(f'{col}_q05'),\n                (pl.col(col).shift(1) * pl.col(col)).mean().alias(f'{col}_autocorr_lag1'),\n                (pl.col(col).shift(2) * pl.col(col)).mean().alias(f'{col}_autocorr_lag2'),\n                (pl.col(col).shift(3) * pl.col(col)).mean().alias(f'{col}_autocorr_lag3'),\n                (pl.col(col).shift(5) * pl.col(col)).mean().alias(f'{col}_autocorr_lag5'),\n                (pl.col(col).ewm_mean(alpha=0.1)).std().alias(f'{col}_trend_std'),\n                (pl.col(col) - pl.col(col).ewm_mean(alpha=0.1)).abs().mean().alias(f'{col}_detrend_mad'),\n                ((pl.col(col) - pl.col(col).shift(1)).abs() > pl.col(col).std()).sum().alias(f'{col}_spike_count'),\n                ((pl.col(col) > pl.col(col).mean()) & \n                (pl.col(col).shift(1) <= pl.col(col).mean()) |\n                (pl.col(col) <= pl.col(col).mean()) & \n                (pl.col(col).shift(1) > pl.col(col).mean())).sum().alias(f'{col}_mean_crossings'),\n                (pl.col(col).diff().fill_null(0).abs()).max().alias(f'{col}_max_velocity'),\n            ])\n            \n            if col != 'rot_w':\n                imu_aggs.extend([\n                    (pl.col(col).abs()).max().alias(f'{col}_abs_max'),\n                    (pl.col(col) < 0).sum().alias(f'{col}_negative_count'),\n                ])\n\n        \n        magnitude_expr = (pl.col('acc_x')**2 + pl.col('acc_y')**2 + pl.col('acc_z')**2).sqrt()\n        imu_aggs.extend([\n            magnitude_expr.mean().alias('acc_magnitude_mean'),\n            magnitude_expr.max().alias('acc_magnitude_max'),\n            magnitude_expr.quantile(0.25).alias('acc_magnitude_q25'),\n            magnitude_expr.diff().fill_null(0).std().alias('acc_magnitude_diff_std'),\n            magnitude_expr.diff().fill_null(0).abs().mean().alias('acc_magnitude_jerk'),\n            \n            (pl.col('acc_x') * pl.col('acc_y')).mean().alias('acc_xy_correlation'),\n            (pl.col('acc_x') * pl.col('acc_z')).mean().alias('acc_xz_correlation'),\n            (pl.col('acc_y') * pl.col('acc_z')).mean().alias('acc_yz_correlation'),\n            \n            (pl.col('acc_x')**2 / (magnitude_expr**2 + 0.001)).mean().alias('acc_x_energy_ratio'),\n            (pl.col('acc_z')**2 / (magnitude_expr**2 + 0.001)).mean().alias('acc_z_energy_ratio'),\n            \n            (magnitude_expr - magnitude_expr.rolling_mean(window_size=10)).abs().mean().alias('acc_dynamic_component'),\n            magnitude_expr.rolling_mean(window_size=10).std().alias('acc_gravity_variation'),\n            \n            (pl.col('acc_y').abs() > pl.col('acc_z').abs()).mean().alias('acc_y_dominance_ratio'),\n            (pl.col('acc_z').abs() > pl.col('acc_x').abs()).mean().alias('acc_z_dominance_ratio'),\n            \n            (pl.col('acc_x').abs().mean() / (pl.col('acc_y').abs().mean() + 0.001)).alias('acc_x_y_ratio'),\n            (pl.col('acc_x').abs().mean() / (pl.col('acc_z').abs().mean() + 0.001)).alias('acc_x_z_ratio'),\n            (pl.col('acc_y').abs().mean() / (pl.col('acc_z').abs().mean() + 0.001)).alias('acc_y_z_ratio'),\n            \n            (pl.col('acc_z') / (magnitude_expr + 0.001)).mean().alias('acc_z_direction_mean'),\n            \n            (pl.col('acc_x').var() / (pl.col('acc_x').var() + pl.col('acc_y').var() + pl.col('acc_z').var() + 0.001)).alias('acc_x_variance_ratio'),\n            (pl.col('acc_y').var() / (pl.col('acc_x').var() + pl.col('acc_y').var() + pl.col('acc_z').var() + 0.001)).alias('acc_y_variance_ratio'),\n            (pl.col('acc_z').var() / (pl.col('acc_x').var() + pl.col('acc_y').var() + pl.col('acc_z').var() + 0.001)).alias('acc_z_variance_ratio'),\n        ])\n        \n        linear_acc_magnitude_expr = (pl.col('linear_acc_x')**2 + pl.col('linear_acc_y')**2 + pl.col('linear_acc_z')**2).sqrt()\n        imu_aggs.extend([\n            linear_acc_magnitude_expr.diff().fill_null(0).std().alias('linear_acc_magnitude_diff_std'),\n            \n            (pl.col('linear_acc_x') * pl.col('linear_acc_z')).mean().alias('linear_acc_xz_correlation'),\n            (pl.col('linear_acc_y') * pl.col('linear_acc_z')).mean().alias('linear_acc_yz_correlation'),\n            \n            (pl.col('linear_acc_x')**2 / (linear_acc_magnitude_expr**2 + 0.001)).mean().alias('linear_acc_x_energy_ratio'),\n            (pl.col('linear_acc_z')**2 / (linear_acc_magnitude_expr**2 + 0.001)).mean().alias('linear_acc_z_energy_ratio'),\n            \n            (pl.col('linear_acc_y').abs() > pl.col('linear_acc_z').abs()).mean().alias('linear_acc_y_dominance_ratio'),\n            (pl.col('linear_acc_z').abs() > pl.col('linear_acc_x').abs()).mean().alias('linear_acc_z_dominance_ratio'),\n            \n            (pl.col('linear_acc_x').abs().mean() / (pl.col('linear_acc_y').abs().mean() + 0.001)).alias('linear_acc_x_y_ratio'),\n            (pl.col('linear_acc_x').abs().mean() / (pl.col('linear_acc_z').abs().mean() + 0.001)).alias('linear_acc_x_z_ratio'),\n            (pl.col('linear_acc_y').abs().mean() / (pl.col('linear_acc_z').abs().mean() + 0.001)).alias('linear_acc_y_z_ratio'),\n            \n            (pl.col('linear_acc_z') / (linear_acc_magnitude_expr + 0.001)).mean().alias('linear_acc_z_direction_mean'),\n            \n            (pl.col('linear_acc_z').var() / (pl.col('linear_acc_x').var() + pl.col('linear_acc_y').var() + pl.col('linear_acc_z').var() + 0.001)).alias('linear_acc_z_variance_ratio'),\n        ])\n        \n        quat_magnitude = (pl.col('rot_w')**2 + pl.col('rot_x')**2 + pl.col('rot_y')**2 + pl.col('rot_z')**2).sqrt()    \n        imu_aggs.extend([        \n            quat_magnitude.mean().alias('rot_magnitude_mean'),\n            quat_magnitude.diff().fill_null(0).std().alias('rot_magnitude_diff_std'),\n            \n            (2 * (pl.col('rot_w') * pl.col('rot_z') + pl.col('rot_x') * pl.col('rot_y'))).mean().alias('rot_euler_z_mean'),\n            (2 * (pl.col('rot_w') * pl.col('rot_x') + pl.col('rot_y') * pl.col('rot_z'))).std().alias('rot_euler_x_std'),\n            (2 * (pl.col('rot_w') * pl.col('rot_y') - pl.col('rot_z') * pl.col('rot_x'))).std().alias('rot_euler_y_std'),\n            (2 * (pl.col('rot_w') * pl.col('rot_z') + pl.col('rot_x') * pl.col('rot_y'))).std().alias('rot_euler_z_std'),\n            \n            (pl.col('rot_w') * pl.col('rot_x')).pow(2).mean().alias('rot_wx_interaction'),\n            (pl.col('rot_w') * pl.col('rot_z')).pow(2).mean().alias('rot_wz_interaction'),\n            \n            (pl.col('rot_y')**2 / (pl.col('rot_x')**2 + pl.col('rot_y')**2 + pl.col('rot_z')**2 + 0.001)).mean().alias('rot_y_axis_dominance'),\n        ])\n\n        part_configs = [\n            ('first_30pct', pl.col('sequence_counter') < pl.max('sequence_counter') * 0.3),\n            ('middle_40pct', (pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.3) & (pl.col('sequence_counter') < pl.max('sequence_counter') * 0.7)),\n            ('last_30pct', pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.7),\n            ('first_half', pl.col('sequence_counter') < pl.max('sequence_counter') * 0.5),\n            ('last_half', pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.5),\n            ('first_10pct', pl.col('sequence_counter') < pl.max('sequence_counter') * 0.1),\n            ('last_10pct', pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.9),\n        ]\n        for part_name, part_expr in part_configs:\n            for col in imu_cols:\n                is_rot_w = 'rot_w' in col\n                imu_aggs.extend([\n                    (pl.when(part_expr).then(pl.col(col))).mean().alias(f'{col}_mean_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).std().alias(f'{col}_std_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).max().alias(f'{col}_max_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).min().alias(f'{col}_min_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).var().alias(f'{col}_var_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.05).alias(f'{col}_q05_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.10).alias(f'{col}_q10_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.25).alias(f'{col}_q25_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.75).alias(f'{col}_q75_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.90).alias(f'{col}_q90_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).entropy().alias(f'{col}_entropy_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).median().alias(f'{col}_median_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).kurtosis().alias(f'{col}_kurtosis_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).skew().alias(f'{col}_skew_{part_name}'),\n                ])\n                \n                if not is_rot_w:\n                    imu_aggs.extend([\n                        (pl.when(part_expr).then(pl.col(col).abs())).mean().alias(f'{col}_abs_mean_{part_name}'),\n                    ])\n\n        for col in imu_cols:\n            imu_aggs.extend([\n                ((pl.when(pl.col('sequence_counter') < pl.max('sequence_counter') * 0.2).then(pl.col(col))).mean() -\n                (pl.when(pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.8).then(pl.col(col))).mean()).alias(f'{col}_early_late_diff'),\n            ])\n            \n        imu_data = dataset.group_by('sequence_id', maintain_order=True).agg(imu_aggs).fill_nan(None).fill_null(strategy=\"forward\").fill_null(strategy=\"forward\").fill_null(0)\n        \n        return imu_data\n\n    def get_thm_data(self, dataset):\n        thm_cols = [col for col in dataset.columns if 'thm_' in col]\n        \n        thm_aggs = []    \n        part_configs = [\n            ('middle_40pct', (pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.3) & (pl.col('sequence_counter') < pl.max('sequence_counter') * 0.7)),\n            ('last_30pct', pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.7),\n            ('last_half', pl.col('sequence_counter') >= pl.max('sequence_counter') * 0.5)\n        ]\n        for part_name, part_expr in part_configs:\n            for col in thm_cols:\n                thm_aggs.extend([\n                    (pl.when(part_expr).then(pl.col(col))).max().alias(f'{col}_max_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).quantile(0.90).alias(f'{col}_q90_{part_name}'),\n                    (pl.when(part_expr).then(pl.col(col))).median().alias(f'{col}_median_{part_name}'),\n                ])\n\n        thm_data = dataset.group_by('sequence_id', maintain_order=True).agg(thm_aggs).fill_nan(None).fill_null(strategy=\"forward\").fill_null(strategy=\"forward\").fill_null(0)                 \n        \n        return thm_data\n\n    def get_tof_data(self, dataset):\n        dataset = dataset.to_pandas()\n        \n        features = {\n            \"sequence_id\": []          \n        }\n        for i in [2, 3, 4]:\n            features[f\"tof_{i}_median\"] = []\n            features[f\"tof_{i}_q25\"] = []\n            features[f\"tof_{i}_cv\"] = []\n            features[f\"tof_{i}_skew\"] = []\n            features[f\"tof_{i}_depth_grad\"] = []\n            features[f\"tof_{i}_depth_smoothness\"] = []\n            \n        for seq_id, seq in dataset.groupby('sequence_id'):\n            seq_df = seq.copy()\n            features[\"sequence_id\"].append(seq_id)\n            for i in [2, 3, 4]:\n                pixel_cols_tof = [f\"tof_{i}_v{p}\" for p in range(64)]\n                tof_sensor_data = seq_df[pixel_cols_tof].replace(-1, np.nan).values.flatten()\n                tof_sensor_data = tof_sensor_data[~np.isnan(tof_sensor_data)]       \n                features[f\"tof_{i}_median\"].append(np.median(tof_sensor_data) if len(tof_sensor_data) > 0 else np.nan)\n                features[f\"tof_{i}_q25\"].append(np.percentile(tof_sensor_data, 25) if len(tof_sensor_data) > 0 else np.nan)\n                features[f\"tof_{i}_cv\"].append(tof_sensor_data.std() / tof_sensor_data.mean() if len(tof_sensor_data) > 0 and tof_sensor_data.mean() != 0 else np.nan)\n                features[f\"tof_{i}_skew\"].append(scipy.stats.skew(tof_sensor_data) if len(tof_sensor_data) > 0 else np.nan)\n                features[f\"tof_{i}_depth_grad\"].append(np.mean(np.abs(np.gradient(tof_sensor_data))) if len(tof_sensor_data) > 1 else np.nan)\n                features[f\"tof_{i}_depth_smoothness\"].append(np.mean(np.abs(np.diff(tof_sensor_data, 2))) if len(tof_sensor_data) > 2 else np.nan)\n                \n        return pl.DataFrame(features).fill_nan(None).fill_null(strategy=\"forward\").fill_null(strategy=\"forward\").fill_null(0)\n        \n    def process_train(self):\n        self.features = joblib.load(f\"{CFG.data_path}/tabular-models/features.pkl\")\n        \n        if os.path.exists(f\"{CFG.data_path}/tabular-models\"):\n            X = joblib.load(f\"{CFG.data_path}/tabular-models/X.pkl\")[self.features]\n            y = joblib.load(f\"{CFG.data_path}/tabular-models/y.pkl\")\n            groups = joblib.load(f\"{CFG.data_path}/tabular-models/groups.pkl\")\n            return X, y, groups\n\n        def _process(dataset):                 \n            imu_data = self.get_imu_data(dataset)\n            thm_data = self.get_thm_data(dataset)    \n            tof_data = self.get_tof_data(dataset)\n            data = imu_data.join(thm_data, on=\"sequence_id\").join(tof_data, on=\"sequence_id\")\n        \n            targets = dataset.group_by('sequence_id', maintain_order=True).agg([\n                pl.col(\"gesture\").first(),\n                pl.col(\"subject\").first(),\n                pl.col(\"sequence_type\").first()\n            ])\n            \n            data = data.join(targets, on=\"sequence_id\", how=\"left\")\n            \n            X = data.drop(['sequence_id', 'gesture', 'subject', 'sequence_type'])\n            y = data.select(pl.col('gesture').map_elements(lambda x: label_to_num[x], return_dtype=pl.Int32))\n            groups = data.select('subject')\n            \n            del dataset, targets\n            gc.collect()\n            \n            return X.to_pandas(), y.to_pandas().values.flatten(), groups.to_pandas().values.flatten()\n\n        train = pl.read_csv(CFG.train_path)\n        train_metadata = pl.read_csv(CFG.train_metadata_path)\n        X_1, y_1, groups_1 = _process(train)\n        \n        thm_tof_cols = [c for c in train.columns if 'thm_' in c or 'tof_' in c]\n        train = train.with_columns([pl.lit(np.nan).alias(c) for c in thm_tof_cols])\n        X_2, y_2, groups_2 = _process(train)\n        \n        X = pd.concat([X_1, X_2]).reset_index(drop=True)[self.features]\n        y = np.concatenate([y_1, y_2])\n        groups = np.concatenate([groups_1, groups_2])\n\n        os.makedirs(\"data/tabular-models\", exist_ok=True)\n        joblib.dump(X, \"data/tabular-models/X.pkl\")\n        joblib.dump(y, \"data/tabular-models/y.pkl\")\n        joblib.dump(groups, \"data/tabular-models/groups.pkl\")\n        \n        del train, train_metadata, X_1, X_2, y_1, y_2, groups_1, groups_2\n        gc.collect()\n        \n        return X, y, groups\n    \n    def process_test(self, dataset, metadata):\n        imu_data = self.get_imu_data(dataset)\n        thm_data = self.get_thm_data(dataset)    \n        tof_data = self.get_tof_data(dataset)\n        data = imu_data.join(thm_data, on=\"sequence_id\").join(tof_data, on=\"sequence_id\")\n        \n        subjects = dataset.group_by('sequence_id', maintain_order=True).agg(pl.col(\"subject\").first())\n        data = data.join(subjects, on=\"sequence_id\", how=\"left\")\n        X_test = data[self.features]\n        \n        del dataset, metadata\n        gc.collect()\n        \n        return X_test.to_pandas()","metadata":{"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gbm_preprocessor = GBMPreprocessor()\nX, y, groups = gbm_preprocessor.process_train()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Inferring on base models","metadata":{}},{"cell_type":"code","source":"def get_scores(y_true, y_pred_probs):\n    y_preds = np.argmax(y_pred_probs, axis=1)\n    \n    binary_f1 = f1_score(\n        np.where(y_true  <= 7, 1, 0),\n        np.where(y_preds <= 7, 1, 0),\n        zero_division = 0.0,\n        average = \"binary\"\n    )\n\n    macro_f1 = f1_score(\n        np.where(y_true   <= 7, y_true, 99),\n        np.where(y_preds  <= 7, y_preds, 99),\n        zero_division = 0.0,\n        average = \"macro\"\n    )\n\n    score = (binary_f1 + macro_f1) / 2\n    \n    return binary_f1, macro_f1, score","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Trainer:\n    def __init__(self, model, config=CFG):\n        self.model = model\n        self.config = config\n\n    def fit(self, X, y, groups=None, fit_args={}, save=False, save_name=None):\n        print(f\"Training {self.model.__class__.__name__}\\n\")\n        \n        fold_scores = {\n            \"binary_f1\": [],\n            \"macro_f1\": [],\n            \"score\": []\n        }      \n        \n        models = []\n        oof_pred_probs = np.zeros((X.shape[0], 18))\n        \n        cv = StratifiedGroupKFold(n_splits=self.config.n_folds, shuffle=True, random_state=self.config.seed)            \n        for fold_idx, (train_idx, val_idx) in enumerate(cv.split(X, y, groups=groups)):\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n            \n            model = clone(self.model)\n            \n            if fit_args:\n                model.fit(X_train, y_train, **fit_args, eval_set=[(X_val, y_val)])\n            else:\n                model.fit(X_train, y_train)\n                \n            models.append(model)\n            \n            y_pred_probs = model.predict_proba(X_val)\n            oof_pred_probs[val_idx] = y_pred_probs\n            \n            binary_f1, macro_f1, score = get_scores(y_val, y_pred_probs)\n            fold_scores[\"binary_f1\"].append(binary_f1)\n            fold_scores[\"macro_f1\"].append(macro_f1)\n            fold_scores[\"score\"].append(score)\n            \n            del model, X_train, y_train, X_val, y_val, y_pred_probs\n            gc.collect()\n            \n            if fit_args:\n                print(f\"\\n--- Fold {fold_idx + 1} - Score: {score:.6f}\\n\\n\")\n            else:\n                print(f\"--- Fold {fold_idx + 1} - Score: {score:.6f}\")\n\n        print(f\"\\n------ Mean Score: {np.mean(fold_scores['score']):.6f} ± {np.std(fold_scores['score']):.6f}\")\n        \n        if save and save_name is not None:\n            self.save(models, oof_pred_probs, save_name, fold_scores)\n\n        return fold_scores\n        \n    def tune(self, X, y, group):\n        fold_scores = []\n        \n        cv = StratifiedGroupKFold(n_splits=self.config.n_folds, shuffle=True, random_state=self.config.seed)            \n        for fold_idx, (train_idx, val_idx) in enumerate(cv.split(X, y, groups=groups)):\n            X_train, X_val = X.iloc[train_idx], X.iloc[val_idx]\n            y_train, y_val = y[train_idx], y[val_idx]\n            \n            model = clone(self.model)\n            model.fit(X_train, y_train)\n            \n            y_pred_probs = model.predict_proba(X_val)\n            \n            _, _, score = get_scores(y_val, y_pred_probs)\n            fold_scores.append(score)\n            \n            del model, X_train, y_train, X_val, y_val, y_pred_probs\n            gc.collect()\n            \n        return np.mean(fold_scores)\n        \n    def save(self, models, oof_pred_probs, save_name, scores):\n        os.makedirs(save_name, exist_ok=True)\n        joblib.dump(models, f\"{save_name}/models.pkl\")\n        joblib.dump(oof_pred_probs, f\"{save_name}/oof_pred_probs.pkl\")\n        joblib.dump(scores, f\"{save_name}/scores_{np.mean(scores['score']):.6f}.pkl\")","metadata":{"trusted":true,"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"oof_pred_probs = {}\nfold_scores = {}","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"nn_model_paths = glob.glob(f\"{CFG.models_path}/neural-networks/**\")\ntabular_model_paths = glob.glob(f\"{CFG.models_path}/tabular-models/*\")\nmodel_paths = nn_model_paths + tabular_model_paths\n\nfor model_path in model_paths:\n    model_name = model_path.split(\"/\")[-1]\n\n    \n    oof_file = joblib.load(f\"{model_path}/oof_pred_probs.pkl\")\n    if \"neural-networks-pt\" not in model_name:\n        oof_file = logit(oof_file)\n    \n    temp_scores = {\n        \"binary_f1\": [],\n        \"macro_f1\": [],\n        \"score\": []\n    }\n    \n    split = StratifiedGroupKFold(n_splits=CFG.n_folds, shuffle=True, random_state=CFG.seed).split(X, y, groups)\n    for _, val_idx in split:\n        y_val = y[val_idx]\n        y_preds = oof_file[val_idx]\n        \n        binary_f1, macro_f1, score = get_scores(y_val, y_preds)\n        \n        temp_scores[\"binary_f1\"].append(binary_f1)\n        temp_scores[\"macro_f1\"].append(macro_f1)\n        temp_scores[\"score\"].append(score)\n        \n    oof_pred_probs[model_name] = oof_file\n    fold_scores[model_name] = temp_scores","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensembling with LogisticRegression","metadata":{}},{"cell_type":"code","source":"X = pd.DataFrame()\n\nfor model_name, predictions in oof_pred_probs.items():\n    X[[f\"{model_name.lower()}-{i}\" for i in range(18)]] = predictions","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def objective(trial):    \n    solver_penalty_options = [\n        ('liblinear', 'l1'),\n        ('liblinear', 'l2'),\n        ('lbfgs', 'l2'),\n        ('lbfgs', None),\n        ('newton-cg', 'l2'),\n        ('newton-cg', None)\n    ]\n    solver, penalty = trial.suggest_categorical('solver_penalty', solver_penalty_options)\n    \n    params = {\n        'random_state': CFG.seed,\n        'max_iter': 1000,\n        'C': trial.suggest_float('C', 0, 10),\n        'tol': trial.suggest_float('tol', 1e-7, 1e-2),\n        'fit_intercept': trial.suggest_categorical('fit_intercept', [True, False]),\n        'solver': solver,\n        'penalty': penalty\n    }\n    \n    model = LogisticRegression(**params)\n    trainer = Trainer(model)\n    return trainer.tune(X, y, groups)\n\nif CFG.run_optuna:\n    sampler = optuna.samplers.TPESampler(seed=CFG.seed, multivariate=True, n_startup_trials=CFG.n_optuna_trials // 10)\n    study = optuna.create_study(direction='maximize', sampler=sampler)\n    study.optimize(objective, n_trials=CFG.n_optuna_trials, n_jobs=-1)\n    best_params = study.best_params\n\n    solver, penalty = best_params['solver_penalty']\n    lr_params = {\n        'random_state': CFG.seed,\n        'max_iter': 1000,\n        'C': best_params['C'],\n        'tol': best_params['tol'],\n        'fit_intercept': best_params['fit_intercept'],\n        'solver': solver,\n        'penalty': penalty\n    }\nelse:\n    lr_params = {\n        'random_state': CFG.seed,\n        'max_iter': 1000,\n        'solver': 'liblinear', \n        'penalty': 'l1', \n        'C': 0.012161263489810276, \n        'tol': 0.005679581289817668, \n        'fit_intercept': False\n    }","metadata":{"_kg_hide-output":true,"scrolled":true,"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(json.dumps(lr_params, indent=2))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lr_model = LogisticRegression(**lr_params) \n\nlr_trainer = Trainer(lr_model)\nfold_scores[\"logistic-regression\"] = lr_trainer.fit(X, y, groups, None, True, \"logistic-regression\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Scores","metadata":{}},{"cell_type":"code","source":"def plot_scores(scores, name):\n    mean_scores = scores.mean().sort_values(ascending=False)\n    order = scores.mean().sort_values(ascending=False).index.tolist()\n\n    min_score = mean_scores.min()\n    max_score = mean_scores.max()\n    padding = (max_score - min_score) * 0.5\n    lower_limit = min_score - padding\n    upper_limit = max_score + padding\n\n    fig, axs = plt.subplots(1, 2, figsize=(15, scores.shape[1] * 0.5))\n\n    boxplot = sns.boxplot(data=scores, order=order, ax=axs[0], orient=\"h\", color=\"grey\")\n    axs[0].set_title(f\"Fold {name}\")\n    axs[0].set_xlabel(\"\")\n    axs[0].set_ylabel(\"\")\n\n    barplot = sns.barplot(x=mean_scores.values, y=mean_scores.index, ax=axs[1], color=\"grey\")\n    axs[1].set_title(f\"Mean {name}\")\n    axs[1].set_xlabel(\"\")\n    axs[1].set_xlim(left=lower_limit, right=upper_limit)\n    axs[1].set_ylabel(\"\")\n\n    for i, (score, model) in enumerate(zip(mean_scores.values, mean_scores.index)):\n        color = \"cyan\" if \"logistic\" in model.lower() else \"grey\"\n        barplot.patches[i].set_facecolor(color)\n        boxplot.patches[i].set_facecolor(color)\n        barplot.text(score, i, round(score, 6), va=\"center\")\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"comp_scores = {}\nmacro_f1_scores = {}\nbinary_f1_scores = {}\n\nfor model in fold_scores.keys():\n    comp_scores[model] = fold_scores[model][\"score\"]\n    macro_f1_scores[model] = fold_scores[model][\"macro_f1\"]\n    binary_f1_scores[model] = fold_scores[model][\"binary_f1\"]\n    \ncomp_scores = pd.DataFrame(comp_scores)\nmacro_f1_scores = pd.DataFrame(macro_f1_scores)\nbinary_f1_scores = pd.DataFrame(binary_f1_scores)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_scores(comp_scores, \"Competition Score\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_scores(macro_f1_scores, \"Macro F1\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plot_scores(binary_f1_scores, \"Binary F1\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}