{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12918439,"sourceType":"datasetVersion","datasetId":8169044}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfrom tqdm import tqdm\n\nimport pandas as pd\nimport numpy as np\nfrom functools import partial\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\n\nfrom scipy.spatial.transform import Rotation as R\nimport pytorch_lightning as pl\n\nfrom colorama import Fore, Style\nc_ = Fore.BLUE\nsr_ = Style.BRIGHT\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:14:20.683227Z","iopub.execute_input":"2025-09-03T08:14:20.683595Z","iopub.status.idle":"2025-09-03T08:14:41.603986Z","shell.execute_reply.started":"2025-09-03T08:14:20.683554Z","shell.execute_reply":"2025-09-03T08:14:41.602903Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CFG:\n    sequence_len = 128\n    num_classes = 18\n    kernel_size = 3\n\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    device_count = torch.cuda.device_count()\n\nprint(f\"{sr_}{c_} => Device is {CFG.device}\")\nprint(f\"{sr_}{c_} => Num GPU of machine is \", CFG.device_count)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:14:41.604965Z","iopub.execute_input":"2025-09-03T08:14:41.605418Z","iopub.status.idle":"2025-09-03T08:14:41.616748Z","shell.execute_reply.started":"2025-09-03T08:14:41.605370Z","shell.execute_reply":"2025-09-03T08:14:41.615686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy import interpolate\n\ndef inter_signal(x):\n    original_len = len(x)\n    interp_func = interpolate.interp1d(np.linspace(0, 1, original_len), x, kind='linear')\n    x_interp = interp_func(np.linspace(0, 1, CFG.sequence_len))\n    return x_interp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:14:41.619264Z","iopub.execute_input":"2025-09-03T08:14:41.619577Z","iopub.status.idle":"2025-09-03T08:14:41.639018Z","shell.execute_reply.started":"2025-09-03T08:14:41.619554Z","shell.execute_reply":"2025-09-03T08:14:41.637945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\nprint(f'{sr_}{c_} Shape of DataFrame: {df.shape}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:14:41.640082Z","iopub.execute_input":"2025-09-03T08:14:41.640449Z","iopub.status.idle":"2025-09-03T08:15:21.631363Z","shell.execute_reply.started":"2025-09-03T08:14:41.640417Z","shell.execute_reply":"2025-09-03T08:15:21.629947Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = df.ffill().bfill().fillna(0.0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:21.633101Z","iopub.execute_input":"2025-09-03T08:15:21.633521Z","iopub.status.idle":"2025-09-03T08:15:37.556090Z","shell.execute_reply.started":"2025-09-03T08:15:21.633485Z","shell.execute_reply":"2025-09-03T08:15:37.555263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_acc_feature = ['acc_x','acc_y','acc_z']\nraw_rot_feature = ['rot_x','rot_y','rot_z','rot_w']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.557154Z","iopub.execute_input":"2025-09-03T08:15:37.557540Z","iopub.status.idle":"2025-09-03T08:15:37.562788Z","shell.execute_reply.started":"2025-09-03T08:15:37.557500Z","shell.execute_reply":"2025-09-03T08:15:37.561810Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def gravity_remove(imu_sequence):\n    \n    acc_values = imu_sequence[raw_acc_feature].values\n    rot_values = imu_sequence[raw_rot_feature].values\n\n    gravity_world = np.array([0,0,9.81])\n    \n    rotation = R.from_quat(rot_values)\n    gravity_sensor = rotation.apply(gravity_world, inverse=True)\n\n    linear_acc = acc_values - gravity_sensor\n    \n    return linear_acc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.563607Z","iopub.execute_input":"2025-09-03T08:15:37.563930Z","iopub.status.idle":"2025-09-03T08:15:37.585930Z","shell.execute_reply.started":"2025-09-03T08:15:37.563900Z","shell.execute_reply":"2025-09-03T08:15:37.584606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_angular_velocity(quat_sequence, sample_rate=128):\n    \n    time_steps = quat_sequence.shape[0]\n    delta_t = 1.0 / sample_rate\n    angular_velocity = np.zeros((time_steps, 3))\n    \n    for i in range(time_steps - 1):\n        q_current = quat_sequence[i]\n        q_next = quat_sequence[i + 1]\n\n        rot_current = R.from_quat(q_current)\n        rot_next = R.from_quat(q_next)\n\n        delta_rot = rot_current.inv() * rot_next\n\n        angular_velocity[i, :] = delta_rot.as_rotvec() / delta_t\n\n    angular_velocity[-1, :] = angular_velocity[-2, :] \n\n    return angular_velocity\n\n\n\ndef compute_angular_distance(quat_sequence):\n \n    seq_length = quat_sequence.shape[0]\n    angular_distance = np.zeros(seq_length)\n    \n    for i in range(seq_length - 1):\n        q_current = quat_sequence[i]\n        q_next = quat_sequence[i + 1]\n\n        r_current = R.from_quat(q_current)\n        r_next = R.from_quat(q_next)\n\n        delta_rot = r_current.inv() * r_next\n        angular_distance[i] = np.linalg.norm(delta_rot.as_rotvec())\n\n    angular_distance[-1] = angular_distance[-2]\n\n    return angular_distance","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.586695Z","iopub.execute_input":"2025-09-03T08:15:37.587092Z","iopub.status.idle":"2025-09-03T08:15:37.599797Z","shell.execute_reply.started":"2025-09-03T08:15:37.587059Z","shell.execute_reply":"2025-09-03T08:15:37.598546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def IMU_Extractor(df):\n\n    ## Gravity Remove\n    linear_acc_df = df.groupby('sequence_id', group_keys=False).apply(\n        lambda df: pd.DataFrame(\n            gravity_remove(df[raw_acc_feature+raw_rot_feature]),\n            columns = raw_acc_feature,\n            index = df.index,\n        )\n    )\n    ## Acc Feature\n    df = df.drop(columns=raw_acc_feature)\n    df[raw_acc_feature] = linear_acc_df\n\n    for axis in ['x', 'y', 'z']:\n        df[f'acc_{axis}_jerk'] = df.groupby('sequence_id')[f'acc_{axis}'].diff().fillna(0)\n\n    df['acc_mag'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n    df['acc_mag_jerk'] = df.groupby('sequence_id')['acc_mag'].diff().fillna(0)\n    df['acc_mag_snap'] = df.groupby('sequence_id')['acc_mag_jerk'].diff().fillna(0)\n\n    ## Rot Feature\n\n    df['rot_angle'] = 2 * np.arccos(df['rot_w'].clip(-1,1))\n    df['rot_angle_vel'] = df.groupby('sequence_id')['rot_angle'].diff().fillna(0)\n    df['rot_angle_snap'] = df.groupby('sequence_id')['rot_angle_vel'].diff().fillna(0)\n\n\n    ## Angular Velocity\n    angular_velocity_df = df.groupby('sequence_id', group_keys=False).apply(\n                lambda df: pd.DataFrame(\n                compute_angular_velocity(df[raw_rot_feature].to_numpy()),\n                columns=['angular_vel_x','angular_vel_y','angular_vel_z'],\n            index=df.index\n            )\n        )\n    df = df.join(angular_velocity_df)\n\n    for axis in ['x', 'y', 'z']:\n        df[f'angular_vel_{axis}_jerk'] = df.groupby('sequence_id')[f'angular_vel_{axis}'].diff().fillna(0)\n\n    df['angular_vel_mag'] = np.sqrt(df['angular_vel_x']**2 + df['angular_vel_y']**2 + df['angular_vel_z']**2)\n    df['angular_vel_mag_jerk'] = df.groupby('sequence_id')['angular_vel_mag'].diff().fillna(0)\n    df['angular_vel_mag_snap'] = df.groupby('sequence_id')['angular_vel_mag_jerk'].diff().fillna(0)\n  \n    ## Angular Distance\n    angular_distance_df = df.groupby('sequence_id', group_keys=False).apply(\n            lambda df: pd.DataFrame(\n                compute_angular_distance(df[raw_rot_feature].to_numpy()),\n                columns=['angular_distance'],\n                index=df.index,\n            )\n    )\n    df = df.join(angular_distance_df)\n\n    ## Hybrid Feature\n    df['acc_angular_vel'] = df['acc_mag'] * df['angular_vel_mag']\n    df['acc_angular_vel_ratio'] = df['acc_mag'] / (df['angular_vel_mag'] + 1e-6)\n    df['acc_angular_distance'] = df['acc_mag'] * df['angular_distance']\n    df['acc_angular_distance_ratio'] = df['acc_mag'] / (df['angular_distance'] + 1e-6)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.603666Z","iopub.execute_input":"2025-09-03T08:15:37.603975Z","iopub.status.idle":"2025-09-03T08:15:37.633921Z","shell.execute_reply.started":"2025-09-03T08:15:37.603952Z","shell.execute_reply":"2025-09-03T08:15:37.632646Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"acc_feature = ['acc_x','acc_y','acc_z',\n               'acc_x_jerk', 'acc_y_jerk', 'acc_z_jerk',\n               'acc_mag','acc_mag_jerk','acc_mag_snap',\n               'acc_angular_vel', 'acc_angular_vel_ratio',  'acc_angular_distance', 'acc_angular_distance_ratio']\n\n\nrot_feature = ['rot_x','rot_y','rot_z','rot_w',\n               'rot_angle', 'rot_angle_vel', 'rot_angle_snap',\n               'angular_vel_x', 'angular_vel_y', 'angular_vel_z', \n               'angular_vel_x_jerk', 'angular_vel_y_jerk', 'angular_vel_z_jerk',\n               'angular_vel_mag', 'angular_vel_mag_jerk', 'angular_vel_mag_snap', \n               'angular_distance']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.634861Z","iopub.execute_input":"2025-09-03T08:15:37.635184Z","iopub.status.idle":"2025-09-03T08:15:37.659633Z","shell.execute_reply.started":"2025-09-03T08:15:37.635156Z","shell.execute_reply":"2025-09-03T08:15:37.658503Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = IMU_Extractor(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:15:37.660828Z","iopub.execute_input":"2025-09-03T08:15:37.661187Z","iopub.status.idle":"2025-09-03T08:16:59.395665Z","shell.execute_reply.started":"2025-09-03T08:15:37.661151Z","shell.execute_reply":"2025-09-03T08:16:59.394704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_thm_feature = ['thm_1','thm_2','thm_3','thm_4','thm_5']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:16:59.396998Z","iopub.execute_input":"2025-09-03T08:16:59.397424Z","iopub.status.idle":"2025-09-03T08:16:59.403868Z","shell.execute_reply.started":"2025-09-03T08:16:59.397363Z","shell.execute_reply":"2025-09-03T08:16:59.402740Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mean_abs_chain(diffs):\n    return sum(d.abs() for d in diffs) / len(diffs)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:16:59.405042Z","iopub.execute_input":"2025-09-03T08:16:59.405379Z","iopub.status.idle":"2025-09-03T08:16:59.431581Z","shell.execute_reply.started":"2025-09-03T08:16:59.405341Z","shell.execute_reply":"2025-09-03T08:16:59.430559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def THM_Extractor(df):\n\n    mask = df['thm_3'] == 0\n    df.loc[mask, 'thm_3'] = np.mean(df.loc[mask, 'thm_2'])\n    \n    # df[raw_thm_feature] = df[raw_thm_feature].apply(\n    # lambda row: row.fillna(\n    #     row.dropna().mean() if row.notna().any() else 0\n    # ),\n    # axis=1\n    # )\n    \n    ## Chain Feature\n    df['thm_right_chain'] = mean_abs_chain([\n        df['thm_3'] - df['thm_2'],\n        df['thm_3'] - df['thm_1'],\n        df['thm_3'] - df['thm_4'],\n    ])\n    df['thm_left_chain'] = mean_abs_chain([\n        df['thm_5'] - df['thm_2'],\n        df['thm_5'] - df['thm_1'],\n        df['thm_5'] - df['thm_4'],\n    ])\n    df['thm_center_chain'] = mean_abs_chain([\n        df['thm_1'] - df['thm_2'],\n        df['thm_1'] - df['thm_3'],\n        df['thm_1'] - df['thm_4'],\n        df['thm_1'] - df['thm_5'],\n\n    ])  \n    df['thm_height_chain'] = mean_abs_chain([\n        df['thm_2'] - df['thm_1'],\n        df['thm_1'] - df['thm_4'],\n    ]) \n\n    \n    ## Chaine Expansion Feature\n    df['thm_right_left_diff'] = df['thm_3'] - df['thm_5']\n    df['thm_top_bottom_diff'] = df[['thm_1','thm_2']].mean(axis=1) - df['thm_4']\n\n    \n    ## Stats Feature\n    df['thm_top_mean'] = df[['thm_1','thm_2']].mean(axis=1)\n    df['thm_middle_mean'] =df[['thm_5','thm_3']].mean(axis=1)\n    df['thm_mean'] = df[raw_thm_feature].mean(axis=1)\n    df['thm_std'] = df[raw_thm_feature].std(axis=1)\n    df['thm_range'] = df[raw_thm_feature].max(axis=1) - df[raw_thm_feature].min(axis=1)\n    \n    return df ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:16:59.432712Z","iopub.execute_input":"2025-09-03T08:16:59.433170Z","iopub.status.idle":"2025-09-03T08:16:59.452951Z","shell.execute_reply.started":"2025-09-03T08:16:59.433139Z","shell.execute_reply":"2025-09-03T08:16:59.451873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"thm_feature = ['thm_1','thm_2','thm_3','thm_4','thm_5',\n               'thm_right_chain','thm_left_chain','thm_center_chain','thm_height_chain',\n               'thm_top_bottom_diff', 'thm_right_left_diff',\n               'thm_top_mean','thm_middle_mean','thm_mean','thm_std','thm_range'\n                   ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:16:59.454071Z","iopub.execute_input":"2025-09-03T08:16:59.454662Z","iopub.status.idle":"2025-09-03T08:16:59.475889Z","shell.execute_reply.started":"2025-09-03T08:16:59.454611Z","shell.execute_reply":"2025-09-03T08:16:59.474805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = THM_Extractor(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:16:59.476995Z","iopub.execute_input":"2025-09-03T08:16:59.477334Z","iopub.status.idle":"2025-09-03T08:17:00.087608Z","shell.execute_reply.started":"2025-09-03T08:16:59.477306Z","shell.execute_reply":"2025-09-03T08:17:00.086574Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_tof_feature = ['tof_1_mean','tof_2_mean','tof_3_mean','tof_4_mean','tof_5_mean']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:00.089014Z","iopub.execute_input":"2025-09-03T08:17:00.089311Z","iopub.status.idle":"2025-09-03T08:17:00.094233Z","shell.execute_reply.started":"2025-09-03T08:17:00.089287Z","shell.execute_reply":"2025-09-03T08:17:00.093452Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tof_list = [col for col in df.columns if 'tof' in col ]\n\ntof_1_list = tof_list[:64]\ntof_2_list = tof_list[64:128]\ntof_3_list = tof_list[128:192]\ntof_4_list = tof_list[192:256]\ntof_5_list = tof_list[256:320]\n\ntof_top_left = []; tof_top_right = [];\ntof_bottom_left = []; tof_bottom_right = []\n\ntof_list = [tof_1_list, tof_2_list, tof_3_list, tof_4_list, tof_5_list]\n\nfor list_ in tof_list: \n\n    tof_top_left.extend([list_[8*i + k] for i in range(0, 4) for k in range(0, 4)])\n    tof_top_right.extend([list_[8*i + k + 4] for i in range(0, 4) for k in range(0, 4)])\n    tof_bottom_left.extend([list_[8*i + k + 32] for i in range(0, 4) for k in range(0, 4)])\n    tof_bottom_right.extend([list_[8*i + k + 36] for i in range(0, 4) for k in range(0, 4)])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:00.095253Z","iopub.execute_input":"2025-09-03T08:17:00.095622Z","iopub.status.idle":"2025-09-03T08:17:00.120589Z","shell.execute_reply.started":"2025-09-03T08:17:00.095590Z","shell.execute_reply":"2025-09-03T08:17:00.119072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def TOF_Extractor(df):\n\n    df[tof_1_list] = df[tof_1_list].replace(-1, 0)\n    df[tof_2_list] = df[tof_2_list].replace(-1, 0)\n    df[tof_3_list] = df[tof_3_list].replace(-1, 0)\n    df[tof_4_list] = df[tof_4_list].replace(-1, 0)\n    df[tof_5_list] = df[tof_5_list].replace(-1, 0)\n    \n    df['tof_1_mean'] = df[tof_1_list].fillna(0).mean(axis=1)\n    df['tof_2_mean'] = df[tof_2_list].fillna(0).mean(axis=1)\n    df['tof_3_mean'] = df[tof_3_list].fillna(0).mean(axis=1)\n    df['tof_4_mean'] = df[tof_4_list].fillna(0).mean(axis=1)\n    df['tof_5_mean'] = df[tof_5_list].fillna(0).mean(axis=1)\n\n    df['tof_1_std'] = df[tof_1_list].fillna(0).std(axis=1)\n    df['tof_2_std'] = df[tof_2_list].fillna(0).std(axis=1)\n    df['tof_3_std'] = df[tof_3_list].fillna(0).std(axis=1)\n    df['tof_4_std'] = df[tof_4_list].fillna(0).std(axis=1)\n    df['tof_5_std'] = df[tof_5_list].fillna(0).std(axis=1)\n\n    df['tof_1_min'] = df[tof_1_list].fillna(0).min(axis=1)\n    df['tof_2_min'] = df[tof_2_list].fillna(0).min(axis=1)\n    df['tof_3_min'] = df[tof_3_list].fillna(0).min(axis=1)\n    df['tof_4_min'] = df[tof_4_list].fillna(0).min(axis=1)\n    df['tof_5_min'] = df[tof_5_list].fillna(0).min(axis=1)\n\n    df['tof_1_max'] = df[tof_1_list].fillna(0).max(axis=1)\n    df['tof_2_max'] = df[tof_2_list].fillna(0).max(axis=1)\n    df['tof_3_max'] = df[tof_3_list].fillna(0).max(axis=1)\n    df['tof_4_max'] = df[tof_4_list].fillna(0).max(axis=1)\n    df['tof_5_max'] = df[tof_5_list].fillna(0).max(axis=1)\n\n\n    df['tof_mean'] = df[raw_tof_feature].mean(axis=1)\n\n    df['tof_top_left_mean'] = df[tof_top_left].fillna(0).mean(axis=1)\n    df['tof_top_right_mean'] = df[tof_top_right].fillna(0).mean(axis=1)\n    df['tof_bottom_left_mean'] = df[tof_bottom_left].fillna(0).mean(axis=1)\n    df['tof_bottom_right_mean'] = df[tof_bottom_right].fillna(0).mean(axis=1)\n\n\n    df['tof_top_diff'] = df['tof_top_left_mean'] - df['tof_top_right_mean']\n    df['tof_bottom_diff'] = df['tof_bottom_left_mean'] - df['tof_bottom_right_mean']\n    df['tof_vertical_diff'] = (df['tof_top_left_mean'] + df['tof_top_right_mean']) - (df['tof_bottom_left_mean'] + df['tof_bottom_right_mean'])\n    df['tof_horizontal_diff'] = (df['tof_top_left_mean'] + df['tof_bottom_left_mean']) - (df['tof_top_right_mean'] + df['tof_bottom_right_mean'])\n\n    df['tof_top_bottom_ratio'] = (df['tof_top_left_mean'] + df['tof_top_right_mean']) / (df['tof_bottom_left_mean'] + df['tof_bottom_right_mean'] + 1e-6)\n\n    df['tof_left_right_ratio'] = (df['tof_top_left_mean'] + df['tof_bottom_left_mean']) / (df['tof_top_right_mean'] + df['tof_bottom_right_mean'] + 1e-6)\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:00.121863Z","iopub.execute_input":"2025-09-03T08:17:00.122158Z","iopub.status.idle":"2025-09-03T08:17:00.159179Z","shell.execute_reply.started":"2025-09-03T08:17:00.122133Z","shell.execute_reply":"2025-09-03T08:17:00.157899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tof_feature = ['tof_1_mean','tof_2_mean','tof_3_mean','tof_4_mean','tof_5_mean',\n               'tof_1_std','tof_2_std','tof_3_std','tof_4_std','tof_5_std',\n               'tof_1_min','tof_2_min','tof_3_min','tof_4_min','tof_5_min',\n               'tof_1_max','tof_2_max','tof_3_max','tof_4_max','tof_5_max',\n               'tof_top_left_mean','tof_top_right_mean','tof_bottom_left_mean','tof_bottom_right_mean',\n               'tof_mean',\n               'tof_top_diff','tof_bottom_diff','tof_vertical_diff','tof_horizontal_diff',\n               'tof_top_bottom_ratio', 'tof_left_right_ratio'\n              ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:00.160310Z","iopub.execute_input":"2025-09-03T08:17:00.160735Z","iopub.status.idle":"2025-09-03T08:17:00.189149Z","shell.execute_reply.started":"2025-09-03T08:17:00.160702Z","shell.execute_reply":"2025-09-03T08:17:00.187330Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = TOF_Extractor(df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:00.190352Z","iopub.execute_input":"2025-09-03T08:17:00.190834Z","iopub.status.idle":"2025-09-03T08:17:26.571037Z","shell.execute_reply.started":"2025-09-03T08:17:00.190806Z","shell.execute_reply":"2025-09-03T08:17:26.569839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"total_feature = acc_feature + rot_feature + thm_feature + tof_feature\n\nprint(f\"{c_}{sr_}=> Acc Feature Count is {len(acc_feature)}\\n\")\nprint(f\"{c_}{sr_}=> Rot Feature Count is {len(rot_feature)}\\n\")\nprint(f\"{c_}{sr_}=> Thm Feature Count is {len(thm_feature)}\\n\")\nprint(f\"{c_}{sr_}=> Tof Feature Count is {len(tof_feature)}\\n\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.572269Z","iopub.execute_input":"2025-09-03T08:17:26.572658Z","iopub.status.idle":"2025-09-03T08:17:26.579236Z","shell.execute_reply.started":"2025-09-03T08:17:26.572628Z","shell.execute_reply":"2025-09-03T08:17:26.578443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CMI_Dataset(Dataset):\n    def __init__(self, data, col):\n        super(CMI_Dataset, self).__init__()\n        self.data = data\n        self.seq_list = data['sequence_id'].unique()\n        self.col = col\n   \n\n    def __len__(self):\n        return len(self.seq_list)\n\n    def __getitem__(self, index):\n\n        X = self.__data_generation(index)\n        X = torch.tensor(X, dtype=torch.float32).permute(1,0)\n        \n        return X\n\n    def __data_generation(self, index):\n        X = np.zeros((CFG.sequence_len,len(self.col)), dtype='float32')\n\n        seq_id = self.seq_list[index]\n        row = self.data[self.data['sequence_id'] == seq_id]\n\n\n        for i, col in enumerate(self.col):\n            signal = inter_signal(row[col].values)\n\n            # normalization\n            if col not in raw_rot_feature:\n                mean = np.mean(signal)\n                eps = 1e-9\n                std = np.std(signal) + eps\n                signal = (signal - mean) / std\n\n            X[:,i] = signal\n\n        return X","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.580620Z","iopub.execute_input":"2025-09-03T08:17:26.581176Z","iopub.status.idle":"2025-09-03T08:17:26.607554Z","shell.execute_reply.started":"2025-09-03T08:17:26.581145Z","shell.execute_reply":"2025-09-03T08:17:26.606224Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Wave_Block(nn.Module):\n    def __init__(self, in_channels, out_channels, dilation_rates, kernel_size=3):\n        super(Wave_Block, self).__init__()\n        self.num_rates = dilation_rates\n        self.convs = nn.ModuleList()\n        self.filter_convs = nn.ModuleList()\n        self.gate_convs = nn.ModuleList()\n        self.convs.append(nn.Conv1d(in_channels, out_channels, kernel_size=1, bias=True))\n        dilation_rates = [2 ** i for i in range(dilation_rates)]\n\n        for dilation_rate in dilation_rates:\n            self.filter_convs.append(\n                nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size,\n                          padding=int((dilation_rate*(kernel_size-1))/2), dilation=dilation_rate)\n            )\n            self.gate_convs.append(\n                nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size,\n                          padding=int((dilation_rate*(kernel_size-1))/2), dilation=dilation_rate)\n            )\n            self.convs.append(\n                nn.Conv1d(out_channels, out_channels, kernel_size=1, bias=True)\n            )\n\n        for i in range(len(self.convs)):\n            nn.init.xavier_uniform_(self.convs[i].weight, gain=nn.init.calculate_gain('relu'))\n            nn.init.zeros_(self.convs[i].bias)\n            \n        for i in range(len(self.filter_convs)):\n            nn.init.xavier_uniform_(self.filter_convs[i].weight, gain=nn.init.calculate_gain('relu'))\n            nn.init.zeros_(self.filter_convs[i].bias)\n            \n        for i in range(len(self.gate_convs)):\n            nn.init.xavier_uniform_(self.gate_convs[i].weight, gain=nn.init.calculate_gain('relu'))\n            nn.init.zeros_(self.gate_convs[i].bias)\n\n    def forward(self,x):\n        x = self.convs[0](x)\n        res = x # skip_connection\n        for i in range(self.num_rates):\n            tanh_out = torch.tanh(self.filter_convs[i](x))\n            sigmoid_out = torch.sigmoid(self.gate_convs[i](x))\n            x = tanh_out * sigmoid_out\n            x = self.convs[i+1](x)\n            res = res + x\n    \n        return res","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.608631Z","iopub.execute_input":"2025-09-03T08:17:26.609001Z","iopub.status.idle":"2025-09-03T08:17:26.624800Z","shell.execute_reply.started":"2025-09-03T08:17:26.608968Z","shell.execute_reply":"2025-09-03T08:17:26.623662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class WaveNet(nn.Module):\n    def __init__(self, kernel_size, in_channels = 1):\n        super(WaveNet, self).__init__()\n\n        self.model = nn.Sequential(\n            Wave_Block(in_channels, 64, 2, kernel_size),\n             nn.MaxPool1d(2),\n             nn.Dropout(0.4),\n             Wave_Block(64, 128, 2, kernel_size),\n             nn.MaxPool1d(2),\n             nn.Dropout(0.2),\n        )\n\n    def forward(self, x):\n        output = self.model(x)\n        return output","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.625889Z","iopub.execute_input":"2025-09-03T08:17:26.626475Z","iopub.status.idle":"2025-09-03T08:17:26.649451Z","shell.execute_reply.started":"2025-09-03T08:17:26.626440Z","shell.execute_reply":"2025-09-03T08:17:26.648550Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class TemporalAttentionPooling(nn.Module):\n    def __init__(self, input_channels, hidden_dim=64):\n        super(TemporalAttentionPooling, self).__init__()\n        self.attn = nn.Sequential(\n            nn.Conv1d(input_channels, hidden_dim, kernel_size=1),\n            nn.Tanh(),\n            nn.Conv1d(hidden_dim, 1, kernel_size=1)\n        )\n\n    def forward(self, x):\n        attn_scores = self.attn(x)\n        attn_weights = F.softmax(attn_scores, dim=-1)\n\n        weighted = x * attn_weights\n        pooled = weighted.sum(dim=-1)\n\n        return pooled","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.650734Z","iopub.execute_input":"2025-09-03T08:17:26.651055Z","iopub.status.idle":"2025-09-03T08:17:26.675783Z","shell.execute_reply.started":"2025-09-03T08:17:26.651029Z","shell.execute_reply":"2025-09-03T08:17:26.674608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class SEBlock(nn.Module):\n    def __init__(self, in_channels, reduction=8):\n        super(SEBlock, self).__init__()\n        self.avgpool = nn.AdaptiveAvgPool1d(1)\n\n        self.mlp = nn.Sequential(\n            nn.Conv1d(in_channels, in_channels//reduction, 1, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Conv1d(in_channels//reduction, in_channels, 1, bias=False),\n            nn.Sigmoid(),\n        )\n\n    def forward(self, input):\n\n        scale = self.avgpool(input)\n        scale = self.mlp(scale)\n        \n\n        return input * scale\n\nclass Residual_Block(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size):\n        super(Residual_Block, self).__init__()\n\n        self.model = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, kernel_size, padding='same', bias=False),\n            nn.BatchNorm1d(out_channels),\n            nn.ReLU(),\n            nn.Conv1d(out_channels, out_channels, kernel_size, padding='same', bias=False),\n            nn.BatchNorm1d(out_channels),\n            nn.ReLU()\n        )\n\n        self.se_block = SEBlock(out_channels)\n\n        self.shortcut = nn.Sequential(\n            nn.Conv1d(in_channels, out_channels, 1, padding='same', bias=False),\n            nn.BatchNorm1d(out_channels)\n        )\n\n        self.dropout = nn.Dropout(0.2)  \n        self.maxpool = nn.MaxPool1d(2)\n        \n    def forward(self, x):\n        shortcut = self.shortcut(x)\n        out = self.model(x)\n        out = self.se_block(out)\n        out = out + shortcut\n        out = F.relu(out)\n        out = self.maxpool(out)\n        out = self.dropout(out)\n\n        return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.682315Z","iopub.execute_input":"2025-09-03T08:17:26.683060Z","iopub.status.idle":"2025-09-03T08:17:26.700692Z","shell.execute_reply.started":"2025-09-03T08:17:26.683023Z","shell.execute_reply":"2025-09-03T08:17:26.699537Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Res_CNN_GRU_Attention(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size):\n        super(Res_CNN_GRU_Attention, self).__init__()\n\n        \n        self.wave_model = WaveNet(kernel_size, in_channels=in_channels)\n\n        self.res_model1 = Residual_Block(in_channels=in_channels, out_channels=out_channels//2, kernel_size=kernel_size)\n        self.res_model2 = Residual_Block(in_channels=out_channels//2, out_channels=out_channels, kernel_size=kernel_size+2)\n\n        self.gru = nn.ModuleList([\n            nn.GRU(input_size=out_channels, hidden_size=out_channels//2, num_layers=1, \n                          batch_first=True, bidirectional=True),\n            nn.GRU(input_size=out_channels, hidden_size=out_channels//2, num_layers=1, \n                          batch_first=True, bidirectional=True),\n        \n            ])\n        self.attn_pool = nn.ModuleList([\n            TemporalAttentionPooling(input_channels=128, hidden_dim=64),\n            TemporalAttentionPooling(input_channels=128, hidden_dim=64),\n        ])\n            \n         \n     \n\n    def forward(self, x):\n        \n        x_trans, x_ges = x[:, :, :CFG.sequence_len//2], x[:, :, CFG.sequence_len//2:]\n\n        ## wave_branch\n        x_trans = self.wave_model(x_trans)\n        gru_trans, _ = self.gru[0](x_trans.permute(0,2,1))\n        trans_out = self.attn_pool[0](gru_trans.permute(0,2,1))\n\n        ## cnn_branch\n        x_ges = self.res_model1(x_ges)\n        x_ges = self.res_model2(x_ges)\n        gru_ges, _ = self.gru[1](x_ges.permute(0,2,1))\n        ges_out = self.attn_pool[1](gru_ges.permute(0,2,1))\n\n        return trans_out, ges_out\n         ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.702138Z","iopub.execute_input":"2025-09-03T08:17:26.702558Z","iopub.status.idle":"2025-09-03T08:17:26.730960Z","shell.execute_reply.started":"2025-09-03T08:17:26.702524Z","shell.execute_reply":"2025-09-03T08:17:26.729559Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CNN_GRU_Attention(nn.Module):\n    def __init__(self, device_feature):\n        super(CNN_GRU_Attention, self).__init__()\n\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n\n        # Wave\n        self.wave_feature = nn.Sequential(\n             Wave_Block(device_feature, 32, 2, 3),\n             nn.MaxPool1d(2),\n             nn.Dropout(0.2),\n             Wave_Block(32, 64, 2, 3),\n             nn.MaxPool1d(2),\n             nn.Dropout(0.2),\n        )\n\n        # 1D CNN\n        self.conv1 = nn.Conv1d(in_channels=device_feature, out_channels=64, kernel_size=3, \n                               padding='same', bias=False)\n        self.batch_norm1 = nn.BatchNorm1d(64)\n        self.relu = nn.ReLU()\n        self.max_pool1 = nn.MaxPool1d(2)\n        self.max_pool2 = nn.MaxPool1d(2)\n        self.dropout1 = nn.Dropout(0.2)\n\n        self.conv2 = nn.Conv1d(in_channels=64, out_channels=128, kernel_size=3, \n                               padding='same', bias=False)\n        self.batch_norm2 = nn.BatchNorm1d(128)\n        self.dropout2 = nn.Dropout(0.2)\n        \n\n        # GRU\n        self.gru = nn.ModuleList([\n            nn.GRU(input_size=64, hidden_size=32, num_layers=1, \n                          batch_first=True, bidirectional=True),\n            nn.GRU(input_size=128, hidden_size=64, num_layers=1, \n                          batch_first=True, bidirectional=True),\n            ])\n        \n        self.attn_pool = nn.ModuleList([\n            TemporalAttentionPooling(input_channels=64, hidden_dim=32),\n            TemporalAttentionPooling(input_channels=128, hidden_dim=64),\n        ])\n            \n\n    def forward(self, x):\n        \n        x_trans, x_ges = x[:, :, :CFG.sequence_len//2], x[:, :, CFG.sequence_len//2:]\n\n        # wave branch with transition part\n        x_trans = self.wave_feature(x_trans)\n\n        gru_trans, _ = self.gru[0](x_trans.permute(0,2,1))\n        trans_out = self.attn_pool[0](gru_trans.permute(0,2,1))\n\n        # cnn_branch with gesture part\n        x_ges = self.conv1(x_ges)\n        x_ges = self.batch_norm1(x_ges)\n        x_ges = self.relu(x_ges)\n        x_ges = self.max_pool1(x_ges)\n        x_ges = self.dropout1(x_ges)\n\n        x_ges = self.conv2(x_ges)\n        x_ges = self.batch_norm2(x_ges)\n        x_ges = self.relu(x_ges)\n        x_ges = self.max_pool2(x_ges)\n        x_ges = self.dropout2(x_ges)\n\n        gru_ges, _ = self.gru[1](x_ges.permute(0,2,1))\n        ges_out = self.attn_pool[1](gru_ges.permute(0,2,1))\n\n        return trans_out, ges_out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.732191Z","iopub.execute_input":"2025-09-03T08:17:26.732577Z","iopub.status.idle":"2025-09-03T08:17:26.760353Z","shell.execute_reply.started":"2025-09-03T08:17:26.732543Z","shell.execute_reply":"2025-09-03T08:17:26.759057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Multi_Model(nn.Module):\n    def __init__(self, kernel_size):\n        super(Multi_Model, self).__init__()\n    \n        self.imu_model = Res_CNN_GRU_Attention(in_channels=len(acc_feature+rot_feature), out_channels=128, kernel_size=3)\n\n        self.thm_tof_model = CNN_GRU_Attention(device_feature = len(thm_feature+tof_feature))\n\n        self.dropout = nn.Dropout(0.2)\n\n    def forward(self, x):\n\n        # acc\n        imu_trans, imu_ges = self.imu_model(x[:,0:30,:])\n\n        # thm + tof\n        thm_tof_trans, thm_tof_ges = self.thm_tof_model(x[:,30:,:])\n\n        \n        trans_out = torch.cat([imu_trans, thm_tof_trans], dim=1)\n        trans_out = self.dropout(trans_out)\n\n        ges_out = torch.cat([imu_ges, thm_tof_ges], dim=1)\n        ges_out = self.dropout(ges_out)\n\n        y = torch.cat([trans_out, ges_out], dim=1)\n\n        return y","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.761598Z","iopub.execute_input":"2025-09-03T08:17:26.761937Z","iopub.status.idle":"2025-09-03T08:17:26.790324Z","shell.execute_reply.started":"2025-09-03T08:17:26.761907Z","shell.execute_reply":"2025-09-03T08:17:26.789002Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class Final_Model(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.feature_extractor = Multi_Model(kernel_size=CFG.kernel_size)\n\n        dummy_input = torch.randn(1, len(total_feature), CFG.sequence_len)\n        with torch.no_grad():\n            dummy_out = self.feature_extractor(dummy_input)\n        feature_dim = dummy_out.shape[1]\n\n        self.head = nn.Sequential(\n            nn.Linear(feature_dim, feature_dim//2, bias=False),\n            nn.BatchNorm1d(feature_dim//2),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(feature_dim//2, CFG.num_classes)\n        )\n\n    def forward(self, x):\n        out = self.feature_extractor(x)\n        out = self.head(out)\n        return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T08:17:26.791349Z","iopub.execute_input":"2025-09-03T08:17:26.791674Z","iopub.status.idle":"2025-09-03T08:17:26.812959Z","shell.execute_reply.started":"2025-09-03T08:17:26.791650Z","shell.execute_reply":"2025-09-03T08:17:26.811873Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Attention Visualization in Method 1\n___","metadata":{}},{"cell_type":"code","source":"def hook_fn(module, input, output, hook_name):\n    x = input[0]\n    attn_scores = module.attn(x)\n    attn_out[hook_name] = F.softmax(attn_scores, dim=-1)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = Final_Model.load_from_checkpoint(\"/kaggle/input/cmi-2025-final-submission/last_fold9.ckpt\")\nmodel.eval()\n\n# IMU branch\nhandle_imu_trans = model.feature_extractor.imu_model.attn_pool[0].register_forward_hook(\n    partial(hook_fn, hook_name=\"imu_trans\")\n)\nhandle_imu_ges = model.feature_extractor.imu_model.attn_pool[1].register_forward_hook(\n    partial(hook_fn, hook_name=\"imu_ges\")\n)\n\n# THM&TOF branch\nhandle_thm_tof_trans = model.feature_extractor.thm_tof_model.attn_pool[0].register_forward_hook(\n    partial(hook_fn, hook_name='thm_tof_trans')\n)\nhandle_thm_tof_ges = model.feature_extractor.thm_tof_model.attn_pool[1].register_forward_hook(\n    partial(hook_fn, hook_name='thm_tof_ges')\n)\n\nseq_list = np.random.choice(df['sequence_id'].unique(), 4)\n\nfor i, seq_id in enumerate(seq_list):\n\n    attn_out = {}\n\n    print(f'{c_}{sr_}#'*25)\n    print(f'### Seqeunce_Id: {seq_id}')\n    print(\"#\"*25)\n    \n    ds = CMI_Dataset(df[df['sequence_id'] == seq_id], col=total_feature)\n    data_loader = DataLoader(ds, shuffle=False, batch_size=1)\n    X = next(iter(data_loader))\n\n    with torch.inference_mode():\n         _ = model(X)\n         \n    fig, axes = plt.subplots(2, 4, figsize=(18, 6))\n    axes = axes.flatten()\n    \n    seq = X[0].permute(1,0).cpu().numpy()\n\n    for i, col in enumerate(raw_acc_feature):\n        axes[0].plot(seq[:CFG.sequence_len//2,i], label=col) \n        axes[1].plot(seq[CFG.sequence_len//2:,i], label=col)\n        \n    for i, col in enumerate(raw_rot_feature):\n        axes[2].plot(seq[:CFG.sequence_len//2,13+i], label=col) \n        axes[3].plot(seq[CFG.sequence_len//2:,13+i], label=col)\n    \n    axes[0].set_title('First 50% in Acc', fontweight='bold'); axes[0].legend(); axes[1].set_title(\"Last 50% in Acc\", fontweight='bold'); axes[1].legend()\n    axes[2].set_title('First 50% in Rot', fontweight='bold'); axes[2].legend(); axes[3].set_title(\"Last 50% in Rot\", fontweight='bold'); axes[3].legend()\n\n    ## imu branch\n    attn_imu_trans = F.interpolate(attn_out['imu_trans'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n    attn_imu_ges = F.interpolate(attn_out['imu_ges'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n\n    axes[0].imshow(\n        np.tile(attn_imu_trans, (len(raw_acc_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,0:3])-2, np.max(seq[:CFG.sequence_len//2,0:3])+2]\n    )\n    axes[1].imshow(\n        np.tile(attn_imu_ges, (len(raw_acc_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[CFG.sequence_len//2:,0:3])-2, np.max(seq[CFG.sequence_len//2:,0:3])+2]\n    )\n    axes[2].imshow(\n        np.tile(attn_imu_trans, (len(raw_rot_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,13:17])-2, np.max(seq[:CFG.sequence_len//2,13:17])+2]\n    )\n    axes[3].imshow(\n        np.tile(attn_imu_ges, (len(raw_rot_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[CFG.sequence_len//2:,13:17])-2, np.max(seq[CFG.sequence_len//2:,13:17])+2]\n    )\n    \n    \n    for i, col in enumerate(raw_thm_feature):\n        axes[4].plot(seq[:CFG.sequence_len//2,30+i], label=col); axes[4].legend() \n        axes[5].plot(seq[CFG.sequence_len//2:,30+i], label=col); axes[5].legend()\n\n    for i, col in enumerate(raw_tof_feature):\n        axes[6].plot(seq[:CFG.sequence_len//2,46+i], label=col); axes[6].legend() \n        axes[7].plot(seq[CFG.sequence_len//2:,46+i], label=col); axes[7].legend()\n    \n    axes[4].set_title('First 50% in THM', fontweight='bold'); axes[5].set_title(\"Last 50% in THM\", fontweight='bold')\n    axes[6].set_title('First 50% in TOF', fontweight='bold'); axes[7].set_title(\"Last 50% in TOF\", fontweight='bold')\n\n    ## thm & tof branch\n    attn_thm_tof_trans = F.interpolate(attn_out['thm_tof_trans'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n    attn_thm_tof_ges = F.interpolate(attn_out['thm_tof_ges'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)     \n\n    axes[4].imshow(\n        np.tile(attn_thm_tof_trans, (len(raw_thm_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,30:35])-2, np.max(seq[:CFG.sequence_len//2,30:35])+2]\n    )\n    axes[5].imshow(\n        np.tile(attn_thm_tof_ges, (len(raw_thm_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,30:35])-2, np.max(seq[:CFG.sequence_len//2,30:35])+2]\n    )\n    axes[6].imshow(\n        np.tile(attn_thm_tof_trans, (len(raw_tof_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,46:51])-2, np.max(seq[:CFG.sequence_len//2,46:51])+2]\n    )\n    axes[7].imshow(\n        np.tile(attn_thm_tof_ges, (len(raw_tof_feature), 1)),\n        aspect='auto',\n        cmap='coolwarm',\n        alpha=0.4,\n        extent=[0, CFG.sequence_len//2-1, np.min(seq[:CFG.sequence_len//2,46:51])-2, np.max(seq[:CFG.sequence_len//2,46:51])+2]\n    )\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T09:16:19.324169Z","iopub.execute_input":"2025-09-03T09:16:19.325359Z","iopub.status.idle":"2025-09-03T09:16:27.828273Z","shell.execute_reply.started":"2025-09-03T09:16:19.325326Z","shell.execute_reply":"2025-09-03T09:16:27.827114Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Attention Visualization in Method 2\n___","metadata":{}},{"cell_type":"code","source":"from matplotlib.collections import LineCollection\nimport matplotlib as mpl\n\ndef plot_sequence_with_attn_colormap(ax, y, attn, cmap=\"viridis\"):\n\n    x = np.arange(len(y))\n\n    points = np.array([x, y]).T.reshape(-1, 1, 2)\n    segments = np.concatenate([points[:-1], points[1:]], axis=1)\n\n    # attn 값으로 색 입힘\n    norm = plt.Normalize(attn.min(), attn.max())\n    lc = LineCollection(segments, cmap=cmap, norm=norm)\n    lc.set_array(attn)   # 색상 매핑 기준 = attention 값\n    lc.set_linewidth(2)\n\n    ax.add_collection(lc)\n    ax.set_xlim(x.min(), x.max())\n    ax.set_ylim(y.min()-1, y.max()+1)\n\n    # colorbar 추가 (attn 기준)\n    sm = mpl.cm.ScalarMappable(norm=norm, cmap=cmap)\n    plt.colorbar(sm, ax=ax, orientation=\"horizontal\", label=\"Attention Weight\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T09:19:19.896658Z","iopub.execute_input":"2025-09-03T09:19:19.897046Z","iopub.status.idle":"2025-09-03T09:19:19.907771Z","shell.execute_reply.started":"2025-09-03T09:19:19.897019Z","shell.execute_reply":"2025-09-03T09:19:19.906815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from functools import partial\n\nmodel = Final_Model.load_from_checkpoint(\"/kaggle/input/cmi-2025-final-submission/last_fold9.ckpt\")\nmodel.eval()\n\n# IMU branch\nhandle_imu_trans = model.feature_extractor.imu_model.attn_pool[0].register_forward_hook(\n    partial(hook_fn, hook_name=\"imu_trans\")\n)\nhandle_imu_ges = model.feature_extractor.imu_model.attn_pool[1].register_forward_hook(\n    partial(hook_fn, hook_name=\"imu_ges\")\n)\n\n# THM&TOF branch\nhandle_thm_tof_trans = model.feature_extractor.thm_tof_model.attn_pool[0].register_forward_hook(\n    partial(hook_fn, hook_name='thm_tof_trans')\n)\nhandle_thm_tof_ges = model.feature_extractor.thm_tof_model.attn_pool[1].register_forward_hook(\n    partial(hook_fn, hook_name='thm_tof_ges')\n)\n\nseq_list = np.random.choice(df['sequence_id'].unique(), 4)\n\nfor i, seq_id in enumerate(seq_list):\n\n    attn_out = {}\n\n    print(f'{c_}{sr_}#'*25)\n    print(f'### Seqeunce_Id: {seq_id}')\n    print(\"#\"*25)\n    \n    ds = CMI_Dataset(df[df['sequence_id'] == seq_id], col=total_feature)\n    data_loader = DataLoader(ds, shuffle=False, batch_size=1)\n    X = next(iter(data_loader))\n\n    with torch.inference_mode():\n         _ = model(X)\n         \n    fig, axes = plt.subplots(2, 4, figsize=(18, 6))\n    axes = axes.flatten()\n    \n    seq = X[0].permute(1,0).cpu().numpy()\n    \n    axes[0].set_title('First 50% in Acc_X', fontweight='bold'); axes[1].set_title(\"Last 50% in Acc_X\", fontweight='bold')\n    axes[2].set_title('First 50% in Rot_X', fontweight='bold'); axes[3].set_title(\"Last 50% in Rot_X\", fontweight='bold')\n\n    ## imu branch\n    attn_imu_trans = F.interpolate(attn_out['imu_trans'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n    attn_imu_ges = F.interpolate(attn_out['imu_ges'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n\n    plot_sequence_with_attn_colormap(axes[0], seq[:CFG.sequence_len//2,0], attn_imu_trans, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[1], seq[CFG.sequence_len//2:,0], attn_imu_ges, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[2], seq[:CFG.sequence_len//2,13], attn_imu_trans, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[3], seq[CFG.sequence_len//2:,13], attn_imu_ges, cmap=\"plasma\")\n    \n    \n    axes[4].set_title('First 50% in THM_1', fontweight='bold'); axes[5].set_title(\"Last 50% in THM_1\", fontweight='bold')\n    axes[6].set_title('First 50% in TOF_1_MEAN', fontweight='bold'); axes[7].set_title(\"Last 50% in TOF_MEAN\", fontweight='bold')\n\n    ## thm & tof branch\n    attn_thm_tof_trans = F.interpolate(attn_out['thm_tof_trans'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n    attn_thm_tof_ges = F.interpolate(attn_out['thm_tof_ges'], size=CFG.sequence_len//2, mode='linear', align_corners=True).numpy().reshape(-1)\n\n    plot_sequence_with_attn_colormap(axes[4], seq[:CFG.sequence_len//2,30], attn_thm_tof_trans, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[5], seq[CFG.sequence_len//2:,30], attn_thm_tof_ges, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[6], seq[:CFG.sequence_len//2,46], attn_thm_tof_trans, cmap=\"plasma\")\n    plot_sequence_with_attn_colormap(axes[7], seq[CFG.sequence_len//2:,46], attn_thm_tof_ges, cmap=\"plasma\")\n\n\n    plt.tight_layout()\n    plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-03T09:33:45.253498Z","iopub.execute_input":"2025-09-03T09:33:45.253867Z","iopub.status.idle":"2025-09-03T09:33:53.693336Z","shell.execute_reply.started":"2025-09-03T09:33:45.253844Z","shell.execute_reply":"2025-09-03T09:33:53.692182Z"}},"outputs":[],"execution_count":null}]}