{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":242561727,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This is the pytorch version of [this great work](https://www.kaggle.com/code/richolson/cmi-2025-1d-cnn-imu-only-baseline) with:\n* preprocessing\n* training\n* validation\n\nThere are two models, one for IMU-only data, and the other for all-feature data","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm.notebook import tqdm\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport gc\nimport pickle\nimport os\nfrom pathlib import Path\nfrom types import SimpleNamespace\nimport shutil\n\nfrom torch.utils.data import DataLoader, Dataset\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.amp import autocast, GradScaler\nfrom torch.optim import lr_scheduler, Adam, AdamW\n\nfrom glob import glob\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nfrom metric import score","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.003198Z","iopub.execute_input":"2025-06-02T12:42:53.003502Z","iopub.status.idle":"2025-06-02T12:42:53.01013Z","shell.execute_reply.started":"2025-06-02T12:42:53.003481Z","shell.execute_reply":"2025-06-02T12:42:53.009192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.environ['CUDA_VISIBLE_DEVICES'] = \"0\"\ncfg = SimpleNamespace(**{})\n\ncfg.fname = 'CMI-001'\ncfg.seed = 42\ncfg.device = torch.device('cuda')\n\ncfg.input_path = Path('/kaggle/input/cmi-detect-behavior-with-sensor-data')\ncfg.save_path = Path('./saved_models')\n\nimc_features = ['acc_x', 'acc_y', 'acc_z']\nall_features = [\n    'acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z', \n    'thm_1', 'thm_2', 'thm_3','thm_4', 'thm_5'\n]\ncfg.imc_index = [0,1,2,3,4,5,6]\ncfg.all_index = [0,1,2,3,4,5,6,7,8,9,10,11] \n# choose which two feature to generate extra features\ncfg.gen_index = [\n    [0, 1],\n    [0, 2],\n    [1, 2]\n]\n# these sequences have only imc features\nimc_only_seq_ids = ['SEQ_000389', 'SEQ_000993', 'SEQ_001905', 'SEQ_002559', 'SEQ_003071', 'SEQ_003221', 'SEQ_004536', 'SEQ_006554', 'SEQ_006816', 'SEQ_007924', 'SEQ_008584', 'SEQ_011032', 'SEQ_012168', 'SEQ_012170', 'SEQ_012619', 'SEQ_013381', 'SEQ_013964', 'SEQ_015093', 'SEQ_016489', 'SEQ_016690', 'SEQ_017041', 'SEQ_017397', 'SEQ_017550', 'SEQ_017726', 'SEQ_018069', 'SEQ_019829', 'SEQ_020153', 'SEQ_020185', 'SEQ_020377', 'SEQ_021335', 'SEQ_022409', 'SEQ_022726', 'SEQ_022927', 'SEQ_024987', 'SEQ_025240', 'SEQ_025685', 'SEQ_027154', 'SEQ_028775', 'SEQ_028871', 'SEQ_029693', 'SEQ_032705', 'SEQ_033061', 'SEQ_033922', 'SEQ_034359', 'SEQ_034994', 'SEQ_035090', 'SEQ_035116', 'SEQ_035309', 'SEQ_035419', 'SEQ_035699', 'SEQ_038052', 'SEQ_039253', 'SEQ_040910', 'SEQ_041113', 'SEQ_041199', 'SEQ_041541', 'SEQ_042079', 'SEQ_042145', 'SEQ_042187', 'SEQ_043744', 'SEQ_044004', 'SEQ_044143', 'SEQ_044475', 'SEQ_045927', 'SEQ_045995', 'SEQ_046244', 'SEQ_047721', 'SEQ_048233', 'SEQ_048511', 'SEQ_048861', 'SEQ_049663', 'SEQ_050020', 'SEQ_050738', 'SEQ_052603', 'SEQ_052680', 'SEQ_052722', 'SEQ_053884', 'SEQ_054261', 'SEQ_055297', 'SEQ_055346', 'SEQ_056071', 'SEQ_056418', 'SEQ_056430', 'SEQ_057193', 'SEQ_057886', 'SEQ_058311', 'SEQ_058580', 'SEQ_058729', 'SEQ_058753', 'SEQ_058923', 'SEQ_059569', 'SEQ_060041', 'SEQ_060392', 'SEQ_061589', 'SEQ_064578', 'SEQ_065179']\n\ncfg.max_length = 188\ncfg.lr = 1e-3\ncfg.num_epochs = 75\ncfg.batch_size = 128\ncfg.workers = 4\ncfg.grad_value = 2.0\ncfg.accumulate = 1\ncfg.decay = 0.01\ncfg.no_decay = True\n\ntargets = [\n    'Above ear - pull hair',\n    'Cheek - pinch skin',\n    'Eyebrow - pull hair',\n    'Eyelash - pull hair',\n    'Forehead - pull hairline',\n    'Forehead - scratch',\n    'Neck - pinch skin',\n    'Neck - scratch',\n    'Drink from bottle/cup',\n    'Feel around in tray and pull out an object',\n    'Glasses on/off',\n    'Pinch knee/leg skin',\n    'Pull air toward your face',\n    'Scratch knee/leg skin',\n    'Text on phone',\n    'Wave hello',\n    'Write name in air',\n    'Write name on leg'\n]\ncfg.label2id = {}\nfor i, k in enumerate(targets):\n    cfg.label2id[k] = i\n\ncfg.num_classes = len(targets)\ncfg.targets = np.array(targets)\n\nrandom.seed(cfg.seed)\nnp.random.seed(cfg.seed)\ntorch.manual_seed(cfg.seed)\ntorch.cuda.manual_seed(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.011457Z","iopub.execute_input":"2025-06-02T12:42:53.012064Z","iopub.status.idle":"2025-06-02T12:42:53.029252Z","shell.execute_reply.started":"2025-06-02T12:42:53.012035Z","shell.execute_reply":"2025-06-02T12:42:53.028194Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## load data","metadata":{}},{"cell_type":"code","source":"demo = pd.read_csv(cfg.input_path / \"train_demographics.csv\")\ntrain = pd.read_csv(cfg.input_path / \"train.csv\")\n\ntrain_agg = train.groupby(['sequence_type','sequence_id','subject','gesture'])['acc_x'].agg(['count']).reset_index()\ntrain_agg = train_agg.rename(columns={'count': 'seq_length'})\ndemo = pd.merge(train_agg, demo, on='subject', how='left').sort_values(['subject','sequence_id']).reset_index(drop=True)\nprint(f\"We have {len(demo)} sequences for training.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.029632Z","iopub.status.idle":"2025-06-02T12:42:53.029856Z","shell.execute_reply.started":"2025-06-02T12:42:53.029743Z","shell.execute_reply":"2025-06-02T12:42:53.029753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"kfold = StratifiedGroupKFold(n_splits=4, shuffle=True, random_state=42)\ndemo['fold'] = -1\nlayers = demo.sequence_type.astype(str) + '/' + demo.handedness.astype(str) + '/' + demo.sex.astype(str)\nfor i, (tr_idx, va_idx) in enumerate(kfold.split(demo, y=layers, groups=demo.subject)):\n    demo.loc[va_idx, 'fold'] = i\n\n# f_name is the file path for Dataset\ndemo['f_name'] = demo['subject'].astype(str) + '_' + demo['sequence_id'].astype(str)\ndemo.fold.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.030765Z","iopub.status.idle":"2025-06-02T12:42:53.031003Z","shell.execute_reply.started":"2025-06-02T12:42:53.030874Z","shell.execute_reply":"2025-06-02T12:42:53.030883Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## calculate the mean/std for normalization","metadata":{}},{"cell_type":"code","source":"for fold in range(4):\n    train_ids = set(demo.loc[demo.fold!=fold, \"sequence_id\"].unique())\n    temp = train[train.sequence_id.isin(train_ids)]\n    print(f\"Fold {fold}, Size: {len(train_ids)}\")\n    all_features_mean = np.nanmean(temp[all_features], axis=0)\n    all_features_std = np.nanstd(temp[all_features], axis=0)\n    np.save(f\"all_features_mean_fold{fold}.npy\", all_features_mean)\n    np.save(f\"all_features_std_fold{fold}.npy\", all_features_std)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.031958Z","iopub.status.idle":"2025-06-02T12:42:53.032285Z","shell.execute_reply.started":"2025-06-02T12:42:53.032114Z","shell.execute_reply":"2025-06-02T12:42:53.032128Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## save input array for each sequence","metadata":{}},{"cell_type":"code","source":"file_path = Path(\"input_arrays\")\nif file_path.exists():\n    shutil.rmtree(file_path)\nfile_path.mkdir(exist_ok=True)\n\nfor i, df in tqdm(train.groupby('sequence_id')):\n    subject = df['subject'].values[0]\n    sequence_id = df['sequence_id'].values[0]\n    for col in all_features:\n        df[col] = df[col].ffill().bfill().fillna(0.)\n    f_name = f\"{subject}_{sequence_id}.npy\"\n    np.save(file_path / f_name, df[all_features].values.astype(np.float32))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.033657Z","iopub.status.idle":"2025-06-02T12:42:53.033931Z","shell.execute_reply.started":"2025-06-02T12:42:53.033811Z","shell.execute_reply":"2025-06-02T12:42:53.033824Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## some functions","metadata":{}},{"cell_type":"code","source":"def pad_sequence(arr, max_len):\n    \"\"\"\n    pad sequences to a fixed length\n    \"\"\"\n    target_shape = (max_len, arr.shape[1])\n    if arr.shape[0] > max_len:\n        return arr[:max_len, :]\n\n    pad_length = max_len - arr.shape[0]\n    padded_arr = np.zeros(target_shape, dtype=arr.dtype)\n    padded_arr[:arr.shape[0], :] = arr\n    return padded_arr\n\n\ndef feature_gen(arr, gen_index):\n    \"\"\"\n    use two features to generate more features\n    for example, acc_x - acc_y --> acc_x_y_diff\n    \"\"\"\n    target_shape = (arr.shape[0], arr.shape[1]+len(gen_index))\n    gen_arr = np.zeros(target_shape, dtype=arr.dtype)\n    gen_arr[:, :arr.shape[1]] = arr\n    for i, ind in enumerate(gen_index):\n        gen_arr[:, arr.shape[1]+i] = arr[:, ind[0]] - arr[:, ind[1]]\n    return gen_arr\n\n\ndef get_optimizer(model, cfg):\n    no_decay = [\"bias\", \"norm\"]\n    if cfg.no_decay:\n        optimizer_parameters = [\n            {'params': [p for n, p in model.named_parameters() \n                        if not any(nd in n for nd in no_decay)],\n             'weight_decay': cfg.decay},\n            {'params': [p for n, p in model.named_parameters() \n                        if any(nd in n for nd in no_decay)],\n             'weight_decay': 0.0},\n        ]\n    else:\n        optimizer_parameters = model.parameters()\n    optimizer = torch.optim.AdamW(optimizer_parameters, lr=cfg.lr)\n    return optimizer\n\n\ndef get_dataloader(dataset, cfg, istrain):\n    dataloader = DataLoader(\n        dataset, \n        batch_size=cfg.batch_size,\n        num_workers=cfg.workers,\n        shuffle=istrain,\n        pin_memory=False,\n        drop_last=istrain\n    )\n    return dataloader","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.140203Z","iopub.execute_input":"2025-06-02T12:42:53.140516Z","iopub.status.idle":"2025-06-02T12:42:53.148935Z","shell.execute_reply.started":"2025-06-02T12:42:53.140493Z","shell.execute_reply":"2025-06-02T12:42:53.148234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMIDataset(Dataset):\n    def __init__(self, df, cfg, feat_mean, feat_std, feature_index):\n        self.cfg = cfg\n        self.targets = df[\"sequence_type\"].values\n        self.labels = df[\"gesture\"].values\n        self.f_names = df[\"f_name\"].values\n        self.seq_lengths = df[\"seq_length\"].values\n        self.label2id = cfg.label2id\n        self.feat_mean = feat_mean\n        self.feat_std = feat_std\n        self.feature_index = feature_index\n\n    def __len__(self):\n        return len(self.labels)\n\n    def __getitem__(self, idx):\n        inputs = np.load(f'/kaggle/working/input_arrays/{self.f_names[idx]}.npy')\n        inputs = (inputs - self.feat_mean) / self.feat_std\n        inputs = inputs[:, self.feature_index]\n        seq_len = self.seq_lengths[idx]\n        \n        if len(inputs) > self.cfg.max_length:\n            inputs = inputs[:self.cfg.max_length, :]\n            seq_len = self.cfg.max_length\n        if len(inputs) < self.cfg.max_length:\n            inputs = pad_sequence(inputs, self.cfg.max_length)\n            seq_len = self.cfg.max_length\n\n        inputs = feature_gen(inputs, self.cfg.gen_index)\n        return {\n            'features': torch.from_numpy(inputs).float(),\n            'targets': torch.tensor(self.label2id[self.labels[idx]], dtype=torch.long),\n            'seq_len': torch.tensor(seq_len, dtype=torch.long)\n        }\n\n\nclass CNN1D(nn.Module):\n    def __init__(self, cfg, feature_dim):\n        super().__init__()\n        self.max_length = cfg.max_length\n        self.feature_dim = feature_dim\n        self.num_classes = cfg.num_classes\n\n        self.block1 = nn.Sequential(\n            nn.Conv1d(self.feature_dim, 256, kernel_size=7, padding=3),\n            nn.BatchNorm1d(256),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.3)\n        )\n        self.block2 = nn.Sequential(\n            nn.Conv1d(256, 384, kernel_size=5, padding=2),\n            nn.BatchNorm1d(384),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.3)\n        )\n        self.block3 = nn.Sequential(\n            nn.Conv1d(384, 512, kernel_size=3, padding=1),\n            nn.BatchNorm1d(512),\n            nn.GELU(),\n            nn.MaxPool1d(kernel_size=2),\n            nn.Dropout(0.4)\n        )\n        self._to_linear = None\n        self._calculate_conv_output((self.feature_dim, self.max_length))\n\n        self.classifier = nn.Sequential(\n            nn.Linear(self._to_linear, 512),\n            nn.GELU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, 256),\n            nn.GELU(),\n            nn.Dropout(0.4),\n            nn.Linear(256, self.num_classes)\n        )\n    \n    def _calculate_conv_output(self, shape):\n        with torch.no_grad():\n            sample = torch.zeros(1, *shape)\n            sample = self.block1(sample)\n            sample = self.block2(sample)\n            sample = self.block3(sample)\n            self._to_linear = sample.view(1, -1).size(1)\n    \n    def forward(self, x):\n        # remember to switch seq_length and num_features in Pytorch!\n        x = x.permute(0, 2, 1)\n        x = self.block1(x)\n        x = self.block2(x)\n        x = self.block3(x)\n        x = x.view(x.size(0), -1)\n        x = self.classifier(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.150237Z","iopub.execute_input":"2025-06-02T12:42:53.150474Z","iopub.status.idle":"2025-06-02T12:42:53.169358Z","shell.execute_reply.started":"2025-06-02T12:42:53.150456Z","shell.execute_reply":"2025-06-02T12:42:53.168781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def batch_to_device(batch, device):\n    return {k:batch[k].to(device, non_blocking=True) for k in batch.keys() if k not in []}\n\n\ndef train_epoch(loader, model, optimizer, scheduler, scaler, device, cfg, epoch):\n    model.train()\n    model.zero_grad()\n    loss_output = []\n    grad_norm_l = []\n    accumulate = cfg.accumulate\n\n    for batch in iter(loader):\n        input_dict = batch_to_device(batch, device)\n        with autocast(device_type=\"cuda\", enabled=True):\n            targets = input_dict['targets']\n            features = input_dict['features']\n            logits = model(features)\n            loss = cfg.loss_func(logits, targets)\n\n        loss_output.append(loss.detach().cpu().item())\n        scaler.scale(loss / cfg.accumulate).backward() \n        accumulate -= 1\n        \n        if accumulate == 0:\n            current_scale = scaler.get_scale()\n            scaler.unscale_(optimizer)\n            nn.utils.clip_grad_value_(model.parameters(), cfg.grad_value)\n            scaler.step(optimizer)     \n            scaler.update()\n            optimizer.zero_grad()\n            accumulate = cfg.accumulate\n\n        scheduler.step()\n\n\ndef valid_epoch(loader, model, device, cfg):\n    model.eval()\n    model.zero_grad()\n    preds_output = []\n    targets_output = []\n    with torch.no_grad():\n        for batch in iter(loader):      \n            input_dict = batch_to_device(batch, device)\n            features = input_dict['features']\n            with autocast(device_type=\"cuda\", enabled=False):\n                preds = model(features)\n                    \n            preds_output.append(preds.detach().cpu())\n            targets = batch['targets']\n            targets_output.append(targets)\n\n    preds_output = torch.cat(preds_output, dim=0)\n    targets_output = torch.cat(targets_output, dim=0)\n    return preds_output.numpy(), targets_output.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.169994Z","iopub.execute_input":"2025-06-02T12:42:53.170214Z","iopub.status.idle":"2025-06-02T12:42:53.189184Z","shell.execute_reply.started":"2025-06-02T12:42:53.170191Z","shell.execute_reply":"2025-06-02T12:42:53.188512Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CMI training part ","metadata":{}},{"cell_type":"code","source":"cfg.loss_func = nn.CrossEntropyLoss()\n\nrecord_oof = []\nrecord_pred = []\nfeature_dim = len(cfg.imc_index) + len(cfg.gen_index)\n\nfor fold in [0, 1, 2, 3]:\n\n    print(f\"*************************** Fold {fold} ***************************\")\n    train_fold = demo[demo.fold != fold].reset_index(drop=True)\n    valid_fold = demo[demo.fold == fold].reset_index(drop=True)\n    \n    train_size = len(train_fold)\n    valid_size = len(valid_fold)\n    print(f\"train size: {train_size}, valid size: {valid_size}\")\n    df_oof = valid_fold[['sequence_id', 'gesture']].copy()\n    df_oof['fold'] = fold\n    df_pred = valid_fold[['sequence_id']].copy()\n    df_pred['fold'] = fold\n    \n    feat_mean = np.load(f\"/kaggle/working/all_features_mean_fold{fold}.npy\")\n    feat_std = np.load(f\"/kaggle/working/all_features_std_fold{fold}.npy\")\n    \n    valid_ds = CMIDataset(valid_fold, cfg, feat_mean, feat_std, cfg.imc_index)\n    valid_dataloader = get_dataloader(valid_ds, cfg, istrain=False)\n\n    model = CNN1D(cfg, feature_dim).to(cfg.device)\n    optimizer = get_optimizer(model, cfg)\n    scaler = GradScaler()\n    scheduler = None\n    result = None\n    best_score = -1\n    for epoch in tqdm(range(cfg.num_epochs), total=cfg.num_epochs):\n        train_ds = CMIDataset(train_fold, cfg, feat_mean, feat_std, cfg.imc_index)\n        train_dataloader = get_dataloader(train_ds, cfg, istrain=True)\n        if scheduler is None:\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                epochs=cfg.num_epochs,\n                steps_per_epoch=len(train_dataloader),\n                pct_start=0.07,\n                anneal_strategy=\"cos\",\n                final_div_factor=0.02,\n            )\n        \n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, cfg.device, cfg, epoch)\n        preds, targets = valid_epoch(valid_dataloader, model, cfg.device, cfg)\n        # from argmax to string label\n        p_index = preds.argmax(1)\n        df_pred['gesture'] = cfg.targets[p_index]\n    \n        result = score(\n            df_oof[['sequence_id', 'gesture']], \n            df_pred[['sequence_id', 'gesture']],\n            row_id_column_name='sequence_id'\n        )\n    \n        if result > best_score:\n            print(f\"Epoch {epoch+1} [{result:.4f}]\")\n            save_path = f'cmi_model_fold{fold}.pt'\n            checkpoint = {\n                'model' : model.state_dict(),\n                'fold' : fold,\n                'best_score': result\n            }\n            torch.save(checkpoint, save_path)\n            best_score = result\n\n    print()\n    record_oof.append(df_oof)\n    record_pred.append(df_pred)\n    \n    del model, optimizer, scheduler, scaler, train_dataloader, valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n\nrecord_oof = pd.concat(record_oof, ignore_index=True)\nrecord_pred = pd.concat(record_pred, ignore_index=True)\nrecord_oof.to_csv(\"cmi_oof.csv\", index=False)\nrecord_pred.to_csv(\"cmi_pred.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-02T12:42:53.190488Z","iopub.execute_input":"2025-06-02T12:42:53.190681Z","execution_failed":"2025-06-02T12:43:25.936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score(\n    record_oof[['sequence_id', 'gesture']], \n    record_pred[['sequence_id', 'gesture']],\n    row_id_column_name='sequence_id'\n)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-06-02T12:43:25.936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## CMI+THM training part ","metadata":{}},{"cell_type":"code","source":"cfg.loss_func = nn.CrossEntropyLoss()\n\nrecord_oof = []\nrecord_pred = []\nfeature_dim = len(cfg.all_index) + len(cfg.gen_index)\n\nfor fold in [0, 1, 2, 3]:\n\n    print(f\"*************************** Fold {fold} ***************************\")\n    train_fold = demo[demo.fold != fold].reset_index(drop=True)\n    valid_fold = demo[demo.fold == fold].reset_index(drop=True)\n    \n    train_size = len(train_fold)\n    valid_size = len(valid_fold)\n    print(f\"train size: {train_size}, valid size: {valid_size}\")\n    df_oof = valid_fold[['sequence_id', 'gesture']].copy()\n    df_oof['fold'] = fold\n    df_pred = valid_fold[['sequence_id']].copy()\n    df_pred['fold'] = fold\n    \n    feat_mean = np.load(f\"/kaggle/working/all_features_mean_fold{fold}.npy\")\n    feat_std = np.load(f\"/kaggle/working/all_features_std_fold{fold}.npy\")\n    \n    valid_ds = CMIDataset(valid_fold, cfg, feat_mean, feat_std, cfg.all_index)\n    valid_dataloader = get_dataloader(valid_ds, cfg, istrain=False)\n\n    model = CNN1D(cfg, feature_dim).to(cfg.device)\n    optimizer = get_optimizer(model, cfg)\n    scaler = GradScaler()\n    scheduler = None\n    result = None\n    best_score = -1\n    for epoch in tqdm(range(cfg.num_epochs), total=cfg.num_epochs):\n        train_ds = CMIDataset(train_fold, cfg, feat_mean, feat_std, cfg.all_index)\n        train_dataloader = get_dataloader(train_ds, cfg, istrain=True)\n        if scheduler is None:\n            scheduler = lr_scheduler.OneCycleLR(\n                optimizer,\n                max_lr=cfg.lr,\n                epochs=cfg.num_epochs,\n                steps_per_epoch=len(train_dataloader),\n                pct_start=0.07,\n                anneal_strategy=\"cos\",\n                final_div_factor=0.02,\n            )\n        \n        train_epoch(train_dataloader, model, optimizer, scheduler, scaler, cfg.device, cfg, epoch)\n        preds, targets = valid_epoch(valid_dataloader, model, cfg.device, cfg)\n        # from argmax to string label\n        p_index = preds.argmax(1)\n        df_pred['gesture'] = cfg.targets[p_index]\n    \n        result = score(\n            df_oof[['sequence_id', 'gesture']], \n            df_pred[['sequence_id', 'gesture']],\n            row_id_column_name='sequence_id'\n        )\n    \n        if result > best_score:\n            print(f\"Epoch {epoch+1} [{result:.4f}]\")\n            save_path = f'cmithm_model_fold{fold}.pt'\n            checkpoint = {\n                'model' : model.state_dict(),\n                'fold' : fold,\n                'best_score': result\n            }\n            torch.save(checkpoint, save_path)\n            best_score = result\n\n    print()\n    record_oof.append(df_oof)\n    record_pred.append(df_pred)\n    \n    del model, optimizer, scheduler, scaler, train_dataloader, valid_dataloader\n    gc.collect()\n    torch.cuda.empty_cache()\n\nrecord_oof = pd.concat(record_oof, ignore_index=True)\nrecord_pred = pd.concat(record_pred, ignore_index=True)\nrecord_oof.to_csv(\"cmithm_oof.csv\", index=False)\nrecord_pred.to_csv(\"cmithm_pred.csv\", index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-06-02T12:43:25.936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"score(\n    record_oof[['sequence_id', 'gesture']], \n    record_pred[['sequence_id', 'gesture']],\n    row_id_column_name='sequence_id'\n)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-06-02T12:43:25.936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"CV: (0.67883 + 0.71494) / 2 = 0.69688\n\nThe inference part might be few days later!","metadata":{}}]}