{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from tqdm import tqdm\n\nimport pandas as pd\nimport numpy as np\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.nn.utils.rnn import pad_sequence, pack_padded_sequence\n\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.preprocessing import StandardScaler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:48:16.694395Z","iopub.execute_input":"2025-07-24T06:48:16.694683Z","iopub.status.idle":"2025-07-24T06:48:25.229767Z","shell.execute_reply.started":"2025-07-24T06:48:16.694654Z","shell.execute_reply":"2025-07-24T06:48:25.228380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"base_path = '/kaggle/input/cmi-detect-behavior-with-sensor-data'\ndata = pd.read_csv(f'{base_path}/train.csv')\n\ntotal_subjects = 81\n\n# Split in train, val, test tramite gli id degli individui: 10 (12%) per il test, 5 (6%) per la val e il 66 (81%) per il train \n\ntest_subjects = sorted(data.subject.value_counts().index)[-10:]\nval_subjects = sorted(data.subject.value_counts().index)[-15:-10]\ntrain_subjects = sorted(data.subject.value_counts().index)[:-15]\nprint(len(test_subjects), len(val_subjects), len(train_subjects))\n\ntrain = data.loc[data['subject'].isin(train_subjects)]\nval = data.loc[data['subject'].isin(val_subjects)]\ntest = data.loc[data['subject'].isin(test_subjects)]\n\ndel data\n\nprint(train.shape, test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:48:25.231712Z","iopub.execute_input":"2025-07-24T06:48:25.232123Z","iopub.status.idle":"2025-07-24T06:49:01.945845Z","shell.execute_reply.started":"2025-07-24T06:48:25.232099Z","shell.execute_reply":"2025-07-24T06:49:01.944901Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_dataset(data):\n\n    not_features_col = [\"row_id\",\"sequence_id\", \"sequence_type\",\"sequence_counter\",\"subject\",\"orientation\",\"behavior\",\"phase\",\"gesture\"]\n    features_col = [c for c in train.columns if c not in not_features_col]\n    target = 'gesture'\n\n    n_gestures = 18\n    \n    gestures_list = ['Forehead - pull hairline', 'Cheek - pinch skin',\n       'Write name on leg', 'Feel around in tray and pull out an object',\n       'Neck - scratch', 'Eyelash - pull hair', 'Eyebrow - pull hair',\n       'Forehead - scratch', 'Above ear - pull hair', 'Wave hello',\n       'Write name in air', 'Neck - pinch skin', 'Text on phone',\n       'Pull air toward your face', 'Pinch knee/leg skin',\n       'Scratch knee/leg skin', 'Drink from bottle/cup', 'Glasses on/off']\n\n    target_map = {g: i for i, g in enumerate(gestures_list)}\n\n    \n    X_ = []\n    y_ = []\n    for sequence in tqdm(data.sequence_id.unique()):\n        sequence_data = data.query(f\"sequence_id == '{sequence}'\")\n        sequence_X = sequence_data[features_col]\n        X_.append(sequence_X)\n        \n        # y = sequence_data[target].apply(lambda x: target_map.get(x))\n        # y_train_one_hot = keras.utils.to_categorical(y, n_gestures)[-1]  # non serve one_hot\n\n        y = sequence_data[target]\n        y_.append(target_map.get(y.iloc[-1]))\n    return X_, y_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:49:01.950702Z","iopub.execute_input":"2025-07-24T06:49:01.950983Z","iopub.status.idle":"2025-07-24T06:49:01.958178Z","shell.execute_reply.started":"2025-07-24T06:49:01.950958Z","shell.execute_reply":"2025-07-24T06:49:01.957140Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_tof(df_list, print_first_img=False):\n    tof_data_list = []\n    for df in tqdm(df_list):\n        tof_cols = [c for c in df.columns if 'tof' in c]\n        tof_data = np.reshape((df[tof_cols].values/255).astype(float), (df.shape[0], len(tof_cols)//8, 8))\n        tof_data_list.append(tof_data)\n        if print_first_img:\n            plt.imshow(tof_data[0])\n    return tof_data_list\n\n\ndef get_sensors(df_list):\n    \"\"\"\n    Return all the sensors as table\n    \"\"\"\n    sens_list = []\n    for df in tqdm(df_list):\n        sens = df[[c for c in df.columns if 'acc' in c or 'rot' in c or 'thm' in c]]\n        sens_list.append(sens)\n    return sens_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:49:01.959153Z","iopub.execute_input":"2025-07-24T06:49:01.959525Z","iopub.status.idle":"2025-07-24T06:49:02.014163Z","shell.execute_reply.started":"2025-07-24T06:49:01.959501Z","shell.execute_reply":"2025-07-24T06:49:02.012994Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, y_train = get_dataset(train)\nX_val, y_val = get_dataset(val)\nX_test, y_test = get_dataset(test)\n\ndel train\ndel val\ndel test","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:49:02.015217Z","iopub.execute_input":"2025-07-24T06:49:02.015568Z","iopub.status.idle":"2025-07-24T06:56:36.609066Z","shell.execute_reply.started":"2025-07-24T06:49:02.015536Z","shell.execute_reply":"2025-07-24T06:56:36.607733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()\nscaler.fit(pd.concat(X_train))\n\nX_train = [pd.DataFrame(scaler.transform(e), columns=e.columns, index=e.index).fillna(0) for e in X_train]\nX_val   = [pd.DataFrame(scaler.transform(e), columns=e.columns, index=e.index).fillna(0) for e in X_val]\nX_test  = [pd.DataFrame(scaler.transform(e), columns=e.columns, index=e.index).fillna(0) for e in X_test]","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GestureSequenceDataset(Dataset):\n    def __init__(self, tof_sequences, sensor_sequences, targets):\n        self.tof_sequences = tof_sequences  # list of (T, H, W)\n        self.sensor_sequences = sensor_sequences  # list of (T, F)\n        self.targets = targets  # list of int\n\n    def __len__(self):\n        return len(self.targets)\n\n    def __getitem__(self, idx):\n        tof = torch.tensor(self.tof_sequences[idx], dtype=torch.float32)  # (T, H, W)\n        sens = torch.tensor(self.sensor_sequences[idx].values, dtype=torch.float32)  # (T, F)\n        label = torch.tensor(self.targets[idx], dtype=torch.long)\n        return tof, sens, label","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:56:36.611781Z","iopub.execute_input":"2025-07-24T06:56:36.612115Z","iopub.status.idle":"2025-07-24T06:56:36.618585Z","shell.execute_reply.started":"2025-07-24T06:56:36.612088Z","shell.execute_reply":"2025-07-24T06:56:36.617605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CNNFeatureExtractor(nn.Module):\n    def __init__(self, out_dim):\n        super().__init__()\n        self.cnn = nn.Sequential(\n            nn.Conv2d(1, 16, 3, padding=1), \n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            \n            nn.Conv2d(16, 32, 3, padding=1), \n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            \n            nn.Flatten(),\n            \n            nn.Linear(32 * 10 * 2, out_dim),  # assuming input 40x8 → pool to 10x5\n            nn.ReLU()\n        )\n\n    def forward(self, x):  # x: (B*T, 1, H, W)\n        return self.cnn(x)\n\n\nclass SensorMLP(nn.Module):\n    def __init__(self, in_dim, out_dim):\n        super().__init__()\n        # print(f\"in_dim: {in_dim}, out_dim: {out_dim}\")\n        self.mlp = nn.Sequential(\n            nn.Linear(in_dim, 64),\n            nn.ReLU(),\n            nn.Linear(64, out_dim),\n            nn.ReLU()\n        )\n\n    def forward(self, x):  # (B*T, F)\n        return self.mlp(x)\n\n\nclass CNN1D(nn.Module):\n    def __init__(self, input_size, output_size):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_channels=input_size, \n                               out_channels=64, \n                               kernel_size=5, \n                               padding=2)\n        self.relu1 = nn.ReLU()\n        self.bn1 = nn.BatchNorm1d(64)\n\n        self.conv2 = nn.Conv1d(in_channels=64, \n                               out_channels=output_size, \n                               kernel_size=5, \n                               padding=2)\n        self.relu2 = nn.ReLU()\n        self.bn2 = nn.BatchNorm1d(output_size)\n\n        self.global_pool = nn.AdaptiveAvgPool1d(1)  # output: (batch, channels, 1)\n\n\n\nclass GestureModel(nn.Module):\n    def __init__(self, img_feat_dim, sens_feat_dim, cnn1d_out=128, num_classes=18):\n        super().__init__()\n        self.cnn_extractor = CNNFeatureExtractor(img_feat_dim)\n        self.sens_mlp = SensorMLP(in_dim=sens_feat_dim, out_dim=img_feat_dim)\n        self.fusion = nn.Linear(img_feat_dim + img_feat_dim, 10)  # compress to 10D\n        self.cnn1d = CNN1D(input_size=10, output_size=cnn1d_out)\n        self.classifier = nn.Linear(cnn1d_out, num_classes)\n\n    def forward(self, x_tof, x_sens):\n        B, T, H, W = x_tof.shape\n        x_tof = x_tof.view(B*T, 1, H, W)\n        x_sens = x_sens.view(B*T, -1)\n\n        tof_feat = self.cnn_extractor(x_tof)\n        sens_feat = self.sens_mlp(x_sens)\n\n        fused = torch.cat([tof_feat, sens_feat], dim=1)\n        compressed = self.fusion(fused)\n        compressed = compressed.view(B, T, -1)\n\n        conv = compressed.permute(0, 2, 1)  # → (batch, features, seq_len) for Conv1d\n        conv = self.cnn1d.relu1(self.cnn1d.bn1(self.cnn1d.conv1(conv)))\n        conv = self.cnn1d.relu2(self.cnn1d.bn2(self.cnn1d.conv2(conv)))\n        conv = self.cnn1d.global_pool(conv).squeeze(-1)  # → (batch, channels)\n        return self.classifier(conv)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T07:11:38.917547Z","iopub.execute_input":"2025-07-24T07:11:38.918517Z","iopub.status.idle":"2025-07-24T07:11:38.933436Z","shell.execute_reply.started":"2025-07-24T07:11:38.918472Z","shell.execute_reply":"2025-07-24T07:11:38.932432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # 1dCNN collate\n# def collate_fn(batch):\n#     sequences, labels = zip(*batch)\n#     lengths = torch.tensor([len(seq) for seq in sequences])\n#     padded_sequences = pad_sequence(sequences, batch_first=True)\n#     labels = torch.tensor(labels, dtype=torch.long)\n#     return padded_sequences, lengths, labels\n\ndef collate_fn(batch):\n    tof_seqs, sens_seqs, labels = zip(*batch)\n\n    tof_padded = pad_sequence([torch.tensor(x) for x in tof_seqs], batch_first=True)  # (B, T, H, W)\n    sens_padded = pad_sequence([torch.tensor(x) for x in sens_seqs], batch_first=True)  # (B, T, F)\n    labels = torch.tensor(labels)\n\n    return tof_padded, sens_padded, labels","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:58:48.722666Z","iopub.execute_input":"2025-07-24T06:58:48.723906Z","iopub.status.idle":"2025-07-24T06:58:48.730210Z","shell.execute_reply.started":"2025-07-24T06:58:48.723864Z","shell.execute_reply":"2025-07-24T06:58:48.728976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BATCH_SIZE = 32\nNUM_FEATURES = X_train[0].shape[1]\nNUM_CLASSES = 18\n\n\ntrain_dataset = GestureSequenceDataset(get_tof(X_train), \n                                       get_sensors(X_train), \n                                       y_train)\nval_dataset   = GestureSequenceDataset(get_tof(X_val), \n                                       get_sensors(X_val), \n                                       y_val)\ntest_dataset   = GestureSequenceDataset(get_tof(X_test), \n                                        get_sensors(X_test), \n                                        y_test)\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collate_fn)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, collate_fn=collate_fn)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False, collate_fn=collate_fn)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:58:58.996849Z","iopub.execute_input":"2025-07-24T06:58:58.997176Z","iopub.status.idle":"2025-07-24T06:59:09.800681Z","shell.execute_reply.started":"2025-07-24T06:58:58.997150Z","shell.execute_reply":"2025-07-24T06:59:09.799738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_eval_metric(model, data_loader):\n\n    gestures_list = ['Forehead - pull hairline', 'Cheek - pinch skin',\n       'Write name on leg', 'Feel around in tray and pull out an object',\n       'Neck - scratch', 'Eyelash - pull hair', 'Eyebrow - pull hair',\n       'Forehead - scratch', 'Above ear - pull hair', 'Wave hello',\n       'Write name in air', 'Neck - pinch skin', 'Text on phone',\n       'Pull air toward your face', 'Pinch knee/leg skin',\n       'Scratch knee/leg skin', 'Drink from bottle/cup', 'Glasses on/off']\n\n    reversed_target_map = {i: g for i, g in enumerate(gestures_list)}\n    \n    binary_map = {\n        'Above ear - pull hair': 1, \n        'Cheek - pinch skin': 1,\n        'Eyelash - pull hair': 1, \n        'Eyebrow - pull hair':1,\n        'Forehead - pull hairline': 1, \n        'Forehead - scratch': 1, \n        'Neck - scratch': 1, \n        'Neck - pinch skin': 1, \n        \n        'Drink from bottle/cup': 0, \n        'Feel around in tray and pull out an object': 0,\n        'Glasses on/off': 0,\n        'Pull air toward your face': 0, \n        'Pinch knee/leg skin': 0,\n        'Scratch knee/leg skin': 0, \n        'Text on phone': 0,\n        'Wave hello': 0,\n        'Write name in air': 0, \n        'Write name on leg': 0, \n    }\n\n    # Evaluation\n    model.eval()\n    all_preds, all_targets = [], []\n    with torch.no_grad():\n        for x_tof, x_sens, y in data_loader:\n            x_tof, x_sens = x_tof.to(device), x_sens.to(device)\n            out = model(x_tof, x_sens)\n            preds = out.argmax(dim=1).cpu().numpy()\n            \n            all_preds.extend(preds)\n            all_targets.extend(y.numpy())\n    \n    acc = np.mean(np.array(all_preds) == np.array(all_targets))\n    f1 = f1_score(all_targets, all_preds, average='macro')\n    \n    # binary score\n    all_text_preds = [reversed_target_map.get(e) for e in all_preds]\n    all_binary_preds = [binary_map.get(e) for e in all_text_preds]\n    \n    all_text_target = [reversed_target_map.get(e) for e in all_targets]\n    all_binary_target = [binary_map.get(e) for e in all_text_target]\n    \n    f1_binary = f1_score(all_binary_target, all_binary_preds)\n\n    print(f\"Test Acc: {acc:.4f} - F1 ('macro'): {f1:.4f} - F1 binary: {f1_binary}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T06:59:43.380975Z","iopub.execute_input":"2025-07-24T06:59:43.381326Z","iopub.status.idle":"2025-07-24T06:59:43.391870Z","shell.execute_reply.started":"2025-07-24T06:59:43.381297Z","shell.execute_reply":"2025-07-24T06:59:43.390887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, device, epochs=50, lr=1e-3):\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n    criterion = nn.CrossEntropyLoss()\n\n    model.to(device)\n\n    for epoch in range(epochs):\n        model.train()\n        total_loss = 0\n        for x_tof, x_sens, y in train_loader:\n            x_tof, x_sens, y = x_tof.to(device), x_sens.to(device), y.to(device)\n\n            optimizer.zero_grad()\n            output = model(x_tof, x_sens)\n            loss = criterion(output, y)\n            loss.backward()\n            optimizer.step()\n            total_loss += loss.item()\n\n        # Evaluation\n        model.eval()\n        all_preds, all_targets = [], []\n        with torch.no_grad():\n            for x_tof, x_sens, y in val_loader:\n                x_tof, x_sens = x_tof.to(device), x_sens.to(device)\n                out = model(x_tof, x_sens)\n                preds = out.argmax(dim=1).cpu().numpy()\n                all_preds.extend(preds)\n                all_targets.extend(y.numpy())\n\n        acc = np.mean(np.array(all_preds) == np.array(all_targets))\n        f1 = f1_score(all_targets, all_preds, average='macro')\n\n        print(f\"Epoch {epoch+1} - Train Loss: {total_loss/len(train_loader):.4f} - Val Acc: {acc:.4f} - F1: {f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T07:00:43.231026Z","iopub.execute_input":"2025-07-24T07:00:43.231360Z","iopub.status.idle":"2025-07-24T07:00:43.241045Z","shell.execute_reply.started":"2025-07-24T07:00:43.231330Z","shell.execute_reply":"2025-07-24T07:00:43.239645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = GestureModel(img_feat_dim=40*8, \n                     sens_feat_dim=12, \n                     cnn1d_out=128, \n                     num_classes=18)\n\ntrain_model(model=model, \n            train_loader=train_loader, \n            val_loader=val_loader, \n            device=device, \n            epochs=500,\n            lr=0.001)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-24T07:11:47.789187Z","iopub.execute_input":"2025-07-24T07:11:47.789569Z","iopub.status.idle":"2025-07-24T07:12:48.567198Z","shell.execute_reply.started":"2025-07-24T07:11:47.789542Z","shell.execute_reply":"2025-07-24T07:12:48.565831Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"get_eval_metric(model, test_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}