{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12959030,"sourceType":"datasetVersion","datasetId":8201441},{"sourceId":259931383,"sourceType":"kernelVersion"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os \nimport random\n\nimport pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport cv2\nfrom PIL import Image\n\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\n\nimport pytorch_lightning as pl\nimport timm ## Pytorch Image Model\n\nfrom colorama import Fore, Style\nc_ = Fore.BLUE\nsr_ = Style.BRIGHT\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:10:31.114799Z","iopub.execute_input":"2025-09-06T08:10:31.115047Z","iopub.status.idle":"2025-09-06T08:10:55.646683Z","shell.execute_reply.started":"2025-09-06T08:10:31.115025Z","shell.execute_reply":"2025-09-06T08:10:55.645707Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    sequence_len = 2048\n    sampling_rate= 256\n    img_size = [256,384]\n    num_classes = 18\n    model_name = 'tf_efficientnet_b0.ns_jft_in1k'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:10:55.647666Z","iopub.execute_input":"2025-09-06T08:10:55.648082Z","iopub.status.idle":"2025-09-06T08:10:55.652582Z","shell.execute_reply.started":"2025-09-06T08:10:55.648053Z","shell.execute_reply":"2025-09-06T08:10:55.651702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy import interpolate\n\ndef inter_signal(x):\n    original_len = len(x)\n    interp_func = interpolate.interp1d(np.linspace(0, 1, original_len), x, kind='linear')\n    x_interp = interp_func(np.linspace(0, 1, CFG.sequence_len))\n\n    return x_interp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:10:55.654429Z","iopub.execute_input":"2025-09-06T08:10:55.654716Z","iopub.status.idle":"2025-09-06T08:10:55.700339Z","shell.execute_reply.started":"2025-09-06T08:10:55.654696Z","shell.execute_reply":"2025-09-06T08:10:55.699464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\nprint(f'{sr_}{c_} Shape of DataFrame: {df.shape}')\nprint(display(df))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:10:55.701305Z","iopub.execute_input":"2025-09-06T08:10:55.701613Z","iopub.status.idle":"2025-09-06T08:11:30.149652Z","shell.execute_reply.started":"2025-09-06T08:10:55.701591Z","shell.execute_reply":"2025-09-06T08:11:30.148644Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"acc_list = ['acc_x', 'acc_y', 'acc_z']\nrot_list = ['rot_x', 'rot_y', 'rot_z', 'rot_w']\n\ntotal_list = acc_list + rot_list","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:11:30.150748Z","iopub.execute_input":"2025-09-06T08:11:30.151090Z","iopub.status.idle":"2025-09-06T08:11:30.155574Z","shell.execute_reply.started":"2025-09-06T08:11:30.151058Z","shell.execute_reply":"2025-09-06T08:11:30.154682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_to_num = {\n    'Above ear - pull hair': 0,  # < ------- TARGETS START\n    'Cheek - pinch skin': 1,\n    'Eyebrow - pull hair': 2,\n    'Eyelash - pull hair': 3,\n    'Forehead - pull hairline': 4,\n    'Forehead - scratch': 5,\n    'Neck - pinch skin': 6,\n    'Neck - scratch': 7,  # < ------- TARGETS END\n    'Drink from bottle/cup': 8,  # < ------- NON-TARGETS START\n    'Feel around in tray and pull out an object': 9,\n    'Glasses on/off': 10,\n    'Pinch knee/leg skin': 11,\n    'Pull air toward your face': 12,\n    'Scratch knee/leg skin': 13,\n    'Text on phone': 14,\n    'Wave hello': 15,\n    'Write name in air': 16,\n    'Write name on leg': 17  # < ------- NON-TARGETS END\n}\n\nnum_to_label = {k: v for k, v in enumerate(label_to_num)}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:11:30.156418Z","iopub.execute_input":"2025-09-06T08:11:30.156684Z","iopub.status.idle":"2025-09-06T08:11:30.186986Z","shell.execute_reply.started":"2025-09-06T08:11:30.156663Z","shell.execute_reply":"2025-09-06T08:11:30.185830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# one hot encoding\nlabel_list = []\n\nfor label, _ in label_to_num.items():\n    df[label] = np.where(df['gesture'] == label, 1.0, 0.0).astype('float32')\n    label_list.append(label)\n\nprint(label_list)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:11:30.187917Z","iopub.execute_input":"2025-09-06T08:11:30.188208Z","iopub.status.idle":"2025-09-06T08:11:30.984022Z","shell.execute_reply.started":"2025-09-06T08:11:30.188185Z","shell.execute_reply":"2025-09-06T08:11:30.983198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import librosa\n\ndef spectrum_from_imu(sequence, sr=CFG.sampling_rate):\n\n    img_acc = np.zeros((64, 64, len(acc_list)),dtype='float32')    \n    img_rot = np.zeros((64, 48, len(rot_list)), dtype='float32')\n    \n    for i, col in enumerate(acc_list):\n        x = sequence[col].values\n\n        x = inter_signal(x)\n        \n        # RAW SPECTROGRAM\n        mel_spec = librosa.feature.melspectrogram(y=x, sr=sr, hop_length=len(x)//64,\n                                                  n_fft=256, n_mels=64, fmin=0, fmax=128)\n        # LOG TRANSFORM\n        width = (mel_spec.shape[1]//8)*8 \n\n        # STANDARDIZE TO -1 TO 1\n        mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n        mel_spec_db = (mel_spec_db+40)/40\n        img_acc[:,:,i] = mel_spec_db\n        \n\n    for i, col in enumerate(rot_list):\n        x = sequence[col].values\n        x = inter_signal(x)\n\n        # RAW SPECTROGRAM\n        mel_spec = librosa.feature.melspectrogram(y=x, sr=sr, hop_length=len(x)//48, \n                                                  n_fft=192, n_mels=64, fmin=0, fmax=128)\n\n        width = (mel_spec.shape[1]//8)*8 \n        # LOG TRANSFORM\n        mel_spec_db = librosa.power_to_db(mel_spec, ref=np.max).astype(np.float32)[:,:width]\n        \n        # STANDARDIZE TO -1 TO 1\n        mel_spec_db = (mel_spec_db+40)/40\n        img_rot[:,:,i] = mel_spec_db\n        \n    return img_acc, img_rot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:11:30.984777Z","iopub.execute_input":"2025-09-06T08:11:30.985095Z","iopub.status.idle":"2025-09-06T08:11:30.993552Z","shell.execute_reply.started":"2025-09-06T08:11:30.985072Z","shell.execute_reply":"2025-09-06T08:11:30.992634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for ges in df['gesture'].unique():\n\n    seq_list = np.random.choice(df[df['gesture'] == ges]['sequence_id'].unique(), 1)\n    print(f'{c_}{sr_}#'*25)\n    print(f\"### Gesture Class: {ges}\")\n    print(\"#\"*25)\n    \n    for i, seq_id in enumerate(seq_list):\n        print(f\"\\n => Sequence_id: {seq_id}\")\n    \n        tmp = df[df['sequence_id'] == seq_id].reset_index()\n        img_acc, img_rot = spectrum_from_imu(tmp, sr=CFG.sampling_rate)\n        \n        \n        plt.figure(figsize=(14,4))\n\n        for i, col in enumerate(total_list):\n\n            if i < len(acc_list): \n                plt.subplot(1,len(total_list),i+1)\n                plt.imshow(img_acc[...,i], aspect='auto', origin='lower', cmap='jet')\n                plt.xlabel('Time'); plt.ylabel('Frequency')\n                plt.title(f'{col}_spectrum', size=10, fontweight='bold')\n\n            else: \n                plt.subplot(1,len(total_list),i+1)\n                plt.imshow(img_rot[...,i-len(acc_list)], aspect='auto', origin='lower', cmap='jet')\n                plt.xlabel('Time'); plt.ylabel('Frequency')\n                plt.title(f'{col}_spectrum', size=10, fontweight='bold')\n\n        plt.tight_layout()\n        plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:11:30.996003Z","iopub.execute_input":"2025-09-06T08:11:30.996286Z","iopub.status.idle":"2025-09-06T08:12:10.454797Z","shell.execute_reply.started":"2025-09-06T08:11:30.996265Z","shell.execute_reply":"2025-09-06T08:12:10.453793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMI_Dataset(Dataset):\n    def __init__(self, data, col):\n        super(CMI_Dataset, self).__init__()\n        self.data = data\n        self.seq_list = data['sequence_id'].unique()\n        self.col = col\n\n    def __len__(self):\n        return len(self.seq_list)\n\n    def __getitem__(self, index):\n\n        X, y = self.__data_generation(index)\n\n        X = torch.tensor(X, dtype=torch.float32).permute(2,0,1)\n        y = torch.tensor(y, dtype=torch.float32)\n\n        return X, y\n\n    def __data_generation(self, index):\n        X = np.zeros((128, 192, 3),dtype='float32')\n        y = np.zeros(CFG.num_classes, dtype='float32')\n        \n        seq_id = self.seq_list[index]\n\n        row = self.data[self.data['sequence_id'] == seq_id]\n\n        img_acc, img_rot = spectrum_from_imu(row, sr=CFG.sampling_rate)\n\n        img_acc = np.concatenate([img_acc[:,:,0], img_acc[:,:,1], img_acc[:,:,2]], axis=1)\n        img_rot = np.concatenate([img_rot[:,:,0], img_rot[:,:,1], img_rot[:,:,2], img_rot[:,:,3]], axis=1)\n\n        img = np.concatenate([img_acc, img_rot], axis=0)\n\n        X = np.tile(img[..., np.newaxis], (1, 1, 3))\n        X = cv2.resize(X, (CFG.img_size[1], CFG.img_size[0]))\n\n        y = row[label_list].values[0]\n        \n        return X, y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:12:10.455676Z","iopub.execute_input":"2025-09-06T08:12:10.455918Z","iopub.status.idle":"2025-09-06T08:12:10.466086Z","shell.execute_reply.started":"2025-09-06T08:12:10.455899Z","shell.execute_reply":"2025-09-06T08:12:10.465034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mask2contour(mask, width=3):\n    w = mask.shape[1]\n    h = mask.shape[0]\n\n    mask2 = np.concatenate([mask[:, width:], np.zeros((h,width))], axis=1)\n    mask2 = np.logical_xor(mask,mask2)\n    \n    mask3 = np.concatenate([mask[width:, :], np.zeros((width,w))], axis=0)\n    mask3 = np.logical_xor(mask, mask3)\n\n    return np.logical_or(mask2, mask3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:12:10.467158Z","iopub.execute_input":"2025-09-06T08:12:10.467467Z","iopub.status.idle":"2025-09-06T08:12:10.484077Z","shell.execute_reply.started":"2025-09-06T08:12:10.467440Z","shell.execute_reply":"2025-09-06T08:12:10.483086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMI_EffNet(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n\n        self.feature_extractor = timm.create_model(\n            CFG.model_name, pretrained=True, features_only=True\n        )\n        self.channels = self.feature_extractor.feature_info[-1]['num_chs']\n        self.dropout = nn.Dropout(0.5)\n\n        self.head = nn.Sequential(\n            nn.Linear(self.channels, self.channels//2, bias=False),\n            nn.BatchNorm1d(self.channels//2),\n            nn.ReLU(),\n            nn.Dropout(0.25),\n            nn.Linear(self.channels//2, CFG.num_classes)\n        )\n\n\n    def forward(self, x):\n        feats = self.feature_extractor(x)  # (B, C, H, W)\n        feats = self.dropout(feats[-1])\n        feats = F.adaptive_avg_pool2d(feats, 1).reshape(x.size(0),-1) # (B, C)\n        out = self.head(feats)\n        \n        return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:12:10.485067Z","iopub.execute_input":"2025-09-06T08:12:10.485721Z","iopub.status.idle":"2025-09-06T08:12:10.498790Z","shell.execute_reply.started":"2025-09-06T08:12:10.485690Z","shell.execute_reply":"2025-09-06T08:12:10.497870Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Grad CAM: Gradient-Weighted Class Activation Mapping","metadata":{}},{"cell_type":"code","source":"model = CMI_EffNet.load_from_checkpoint(\"/kaggle/input/cmi-2025-efficientnet/last_fold2.ckpt\")\n\nactivations = {}\ngrads = {}\n\ndef forward_hook(module, input, output):\n    activations[\"value\"] = output.detach()\n\ndef backward_hook(module, grad_input, grad_output):\n    grads[\"value\"] = grad_output[0]  # grad_output은 tuple\n\n\ntarget_layer_name = model.feature_extractor.feature_info[-1]['module']\ntarget_layer = dict(model.feature_extractor.named_modules())[target_layer_name]\n\nhandle_f = target_layer.register_forward_hook(forward_hook)\nhandle_b = target_layer.register_full_backward_hook(backward_hook)\n\n\nmodel.eval()\ngclahe = cv2.createCLAHE(clipLimit=16.0, tileGridSize=(8,8))\n\nfor i, ges in enumerate(df['gesture'].unique()):\n\n    print(f\"{c_}{sr_}#\"*25)\n    print(f'### Gesture: {ges}')\n    print(\"#\"*25)\n    print(\"\\n\")\n    \n    tmp = df[df['gesture'] == ges].reset_index(drop=True)\n    ds = CMI_Dataset(tmp, col=total_list)\n    data_loader = DataLoader(ds, shuffle=False, batch_size=8)\n    \n    fig, axes = plt.subplots(2, 4, figsize=(14, 8))\n    axes = axes.flatten()\n\n    imgs, tars = next(iter(data_loader))\n\n    preds = model(imgs)  # forward pass\n\n    target = (preds * tars).sum() \n    model.zero_grad()\n    target.backward() \n    \n\n    for k in range(imgs.size(0)):\n\n        img = imgs[k].detach().cpu().permute(1,2,0).numpy()[...,0]\n        img -= img.min()\n        img /= img.max()\n        img = (img * 255).astype('uint8')\n        img = gclahe.apply(img)\n    \n        feature_map = activations[\"value\"][k].unsqueeze(0)  # (1,C,H,W)\n        grad_map = grads[\"value\"][k].unsqueeze(0)           # (1,C,H,W)\n\n        weights = grad_map.mean(dim=(2,3), keepdim=True)    # (1,C,1,1)\n        cam = F.relu((weights * feature_map).sum(dim=1, keepdim=True))  # (1,1,H,W)\n        cam = F.interpolate(cam, size=CFG.img_size, mode='bilinear', align_corners=False)\n\n        cam = cam.squeeze().detach().cpu().numpy()\n        cam = (cam - cam.min()) / (cam.max() - cam.min() + 1e-8)  # normalize 0~1\n\n        cam[cam >= 0.7] = 100\n        cam[cam < 0.7] = 0\n        cam = mask2contour(cam)\n\n        mx = np.max(img)\n\n        img[cam > 0] = mx\n\n        axes[k].imshow(img, aspect='auto', origin='lower')\n        axes[k].axis('off')\n    \n    plt.tight_layout()\n    plt.show()\n\n# Hook 제거\nhandle_f.remove()\nhandle_b.remove()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-06T08:22:50.146988Z","iopub.execute_input":"2025-09-06T08:22:50.147286Z","iopub.status.idle":"2025-09-06T08:23:37.417126Z","shell.execute_reply.started":"2025-09-06T08:22:50.147265Z","shell.execute_reply":"2025-09-06T08:23:37.416098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}