{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12351023,"sourceType":"datasetVersion","datasetId":7665416,"isSourceIdPinned":true},{"sourceId":12519360,"sourceType":"datasetVersion","datasetId":7699074},{"sourceId":242954653,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Setup environment (CPU/GPU/TPU) and import packages","metadata":{}},{"cell_type":"code","source":"## For TPU environment (install missing packages / reinstall tensorflow to solve NaN topic during training / restart kernel)\n\nimport IPython\nimport tensorflow as tf\n\nif len(tf.config.experimental.list_logical_devices('TPU'))>0:\n    !pip install -q tensorflow-tpu -f https://storage.googleapis.com/libtpu-tf-releases/index.html --force-reinstall\n    !pip install -q pydot\n    !pip install -q -U keras-tuner\n    !pip install xgboost\n    IPython.Application.instance().kernel.do_shutdown(True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T13:39:12.353196Z","iopub.execute_input":"2025-07-19T13:39:12.353601Z","iopub.status.idle":"2025-07-19T13:41:12.622672Z","shell.execute_reply.started":"2025-07-19T13:39:12.353576Z","shell.execute_reply":"2025-07-19T13:41:12.617736Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Import packages\n\nimport os\nimport time\nimport polars as pl\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport keras_tuner as kt\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import imshow\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom tensorflow.keras.regularizers import L2\nprint('Tensorflow version: ' + tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:34:48.231041Z","iopub.execute_input":"2025-07-19T20:34:48.231572Z","iopub.status.idle":"2025-07-19T20:35:01.925064Z","shell.execute_reply.started":"2025-07-19T20:34:48.231545Z","shell.execute_reply":"2025-07-19T20:35:01.924433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Detect hardware (CPU/GPU/TPU), setup environment and return appropriate distribution strategy\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu='local') # set tpu is local as it should be available in the VM\n    print('✅ Running on TPU ', tpu.master())\nexcept:\n    print('❌ Using CPU/GPU')\n    tpu = None\n\nif tpu:\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:35:02.872368Z","iopub.execute_input":"2025-07-19T20:35:02.872635Z","iopub.status.idle":"2025-07-19T20:35:03.109493Z","shell.execute_reply.started":"2025-07-19T20:35:02.872609Z","shell.execute_reply":"2025-07-19T20:35:03.108854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Load and preprocess data into datasets","metadata":{}},{"cell_type":"code","source":"## Read csv files\n\npath = '/kaggle/input/cmi-detect-behavior-with-sensor-data/'\ntrainval = pd.read_csv(path + \"train.csv\")\ntrainval_demographics = pd.read_csv(path + \"train_demographics.csv\")\ntest = pd.read_csv(path + \"test.csv\")\ntest_demographics = pd.read_csv(path + \"test_demographics.csv\")\n#trainval = trainval[:50000] # To save time during debugging/expermenting","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:35:03.110890Z","iopub.execute_input":"2025-07-19T20:35:03.111106Z","iopub.status.idle":"2025-07-19T20:35:30.516838Z","shell.execute_reply.started":"2025-07-19T20:35:03.111087Z","shell.execute_reply":"2025-07-19T20:35:30.516084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Spliting the train data into train and val\n\ncv = StratifiedGroupKFold(n_splits=10)\nfor fold, (train_idx, val_idx) in enumerate(cv.split(trainval, trainval['gesture'], groups=np.array(trainval['sequence_id'], dtype='<U10'))):\n    print(train_idx)\n    print(val_idx)\n    train, val = trainval.iloc[train_idx], trainval.iloc[val_idx]\n    print(f\"✅ Fold {fold}: Train size = {len(train_idx)}, Val size = {len(val_idx)}\")\n    break  # Use only the first fold for now","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:35:30.517727Z","iopub.execute_input":"2025-07-19T20:35:30.517988Z","iopub.status.idle":"2025-07-19T20:35:37.465061Z","shell.execute_reply.started":"2025-07-19T20:35:30.517969Z","shell.execute_reply":"2025-07-19T20:35:37.464381Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Generate transformation dictionaries for classes\n\nclass2bfrb = {'Cheek - pinch skin': 1, 'Forehead - pull hairline': 1, 'Write name on leg': 0,\n              'Feel around in tray and pull out an object': 0, 'Neck - scratch': 1,\n              'Neck - pinch skin': 1, 'Eyelash - pull hair': 1, 'Eyebrow - pull hair': 1,\n              'Forehead - scratch': 1, 'Above ear - pull hair': 1, 'Wave hello': 0, 'Write name in air': 0,\n              'Text on phone': 0, 'Pull air toward your face': 0, 'Pinch knee/leg skin': 0,\n              'Scratch knee/leg skin': 0, 'Drink from bottle/cup': 0, 'Glasses on/off': 0}\nclass2id = {key: i for i, key in enumerate(class2bfrb.keys())}\nid2class = {i: value for i, value in enumerate(class2bfrb.keys())}\nid2bfrb = {i: value for i, value in enumerate(class2bfrb.values())}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:35:37.465774Z","iopub.execute_input":"2025-07-19T20:35:37.465989Z","iopub.status.idle":"2025-07-19T20:35:37.471143Z","shell.execute_reply.started":"2025-07-19T20:35:37.465967Z","shell.execute_reply":"2025-07-19T20:35:37.470397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract sensor related data\n\n# scalers = {'scaler_acc' : MinMaxScaler(feature_range=(-1, 1)).fit([[-15],[15]]),\n#            'scaler_rot_w' : MinMaxScaler().fit([[0],[1]]), 'scaler_rot' : MinMaxScaler().fit([[-1],[1]]),\n#            'scaler_tof' : MinMaxScaler().fit([[-1],[249]]), 'scaler_thm' : MinMaxScaler().fit([[21],[35]])}\nscalers = {'scaler_acc' : MinMaxScaler().fit([[-50],[50]]), 'scaler_rot' : MinMaxScaler().fit([[-1],[1]]),\n           'scaler_tof' : MinMaxScaler().fit([[-1],[249]]), 'scaler_thm' : MinMaxScaler().fit([[-1],[40]])}\n\nseq_len=80\n\ndef extract_time_series_data(data, training):\n    start_time = time.time()\n    poss_tensor_padded_list = []\n    for i in range(1, 6):\n       globals()[f'tof_thm_tensor_padded_list{i}'] = []\n    \n    for s, sequence_id in enumerate(data['sequence_id'].unique()):\n        if s%1000 == 0: # Show progress\n            start_time2 = time.time()\n        \n        # Filter data for one sequence and truncate if longer than seq_len\n        data_seq = data[data['sequence_id'] == sequence_id].ffill().bfill().fillna(0)\n        if len(data_seq.index) > seq_len:\n            data_seq = data_seq[-seq_len:]\n            \n        # Extract position/orientation related data\n        pos_tensor_padded_list = []\n        for column in ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']:\n            pos_array  = pd.DataFrame(data_seq, columns=[column]).to_numpy()\n            if column in ['acc_x', 'acc_y', 'acc_z']:\n                pos_array = scalers['scaler_acc'].transform(pos_array)\n            pos_tensor = tf.convert_to_tensor(pos_array.reshape(1,1,-1,1), dtype=tf.float32)\n            pad_length = seq_len - pos_tensor.shape[2] # Max length in train dataset is 71 min length is 3 after removing transitions\n            paddings_2d = tf.constant([[0, 0,], [0, 0], [pad_length, 0], [0, 0]])\n            pos_tensor_padded = tf.pad(pos_tensor, paddings_2d, \"CONSTANT\")  # Shape (1,1,seq_len,1)\n            pos_tensor_padded_list.append(pos_tensor_padded)\n        poss_tensor_padded = tf.concat(pos_tensor_padded_list, axis=1) # Shape (1,7,seq_len,1)\n        poss_tensor_padded_list.append(poss_tensor_padded)\n\n        # Extract ToF/THM related data\n        for i in range(1, 6):\n            # Extract ToF data\n            image_arrays  = pd.DataFrame(data_seq, columns=[f'tof_{i}_v{j}' for j in range(64)]).to_numpy()\n            image_arrays = (image_arrays + 1)/250\n            image3d_tensor = tf.convert_to_tensor(image_arrays.reshape(1,8,8,-1,1), dtype=tf.float32)\n            paddings_3d = tf.constant([[0, 0,], [0, 0], [0, 0], [pad_length, 0], [0, 0]])\n            image3d_tensor_padded = tf.pad(image3d_tensor, paddings_3d, \"CONSTANT\")  # Shape (1,8,8,seq_len,1)\n            # Extract THM data\n            temp_array  = pd.DataFrame(data_seq, columns=[f'thm_{i}']).to_numpy()\n            temp_array = scalers['scaler_thm'].transform(temp_array)\n            temp_tensor = tf.convert_to_tensor(temp_array.reshape(1,1,1,-1,1), dtype=tf.float32)\n            multiply_factor = tf.constant([1,8,8,1,1], tf.int32)\n            temp_tensor = tf.tile(temp_tensor, multiply_factor)\n            temp_tensor_padded = tf.pad(temp_tensor, paddings_3d, \"CONSTANT\") # Shape (1,8,8,seq_len,1)\n            tof_thm_tensor_padded = tf.concat([image3d_tensor_padded, temp_tensor_padded], axis=4) # Shape (1,8,8,seq_len,2)\n            globals()[f'tof_thm_tensor_padded_list{i}'].append(tof_thm_tensor_padded)\n        if s%1000 == 0: print(\"Time taken for 1000 seqs: %.2fs\" % (time.time() - start_time2))\n    \n    data_tensor_0 = tf.concat(poss_tensor_padded_list, axis=0)\n    for i in range(1, 6):\n        globals()[f'data_tensor_{i}'] = tf.concat(globals()[f'tof_thm_tensor_padded_list{i}'], axis=0)\n    print(\"Time taken: %.2fs\" % (time.time() - start_time))\n    return data_tensor_0, data_tensor_1, data_tensor_2, data_tensor_3, data_tensor_4, data_tensor_5\n\ntrain_tensor_0, train_tensor_1, train_tensor_2, train_tensor_3, train_tensor_4, train_tensor_5 = extract_time_series_data(train, True)\nval_tensor_0, val_tensor_1, val_tensor_2, val_tensor_3, val_tensor_4, val_tensor_5 = extract_time_series_data(val, False)\ntest_tensor_0, test_tensor_1, test_tensor_2, test_tensor_3, test_tensor_4, test_tensor_5 = extract_time_series_data(test, False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:35:37.472632Z","iopub.execute_input":"2025-07-19T20:35:37.473017Z","iopub.status.idle":"2025-07-19T20:42:02.227287Z","shell.execute_reply.started":"2025-07-19T20:35:37.472999Z","shell.execute_reply":"2025-07-19T20:42:02.226506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract Subject related data\n\nscalers_sub = {'scaler_age' : StandardScaler(), 'scaler_height_cm' : StandardScaler(),\n               'scaler_shoulder_to_wrist_cm' : StandardScaler(), 'scaler_elbow_to_wrist_cm' : StandardScaler()}\n\ndef extract_subject_related_data(data, data_demographics, scalers, training):\n    # Filter subject related data that every sequence id apperas only once\n    data_fil = data[['sequence_id', 'subject']].drop_duplicates(subset=['sequence_id'])\n    data_fil = data_fil.merge(data_demographics, how='left', left_on='subject', right_on='subject')\n    \n    # Scaling numeric values\n    data_fil[['adult_child', 'sex', 'handedness']] = data_fil[['adult_child', 'sex', 'handedness']].astype(dtype=np.float32)\n    if training:\n        scalers['scaler_age'].fit(data_fil[['age']])\n        scalers['scaler_height_cm'].fit(data_fil[['height_cm']])\n        scalers['scaler_shoulder_to_wrist_cm'].fit(data_fil[['shoulder_to_wrist_cm']])\n        scalers['scaler_elbow_to_wrist_cm'].fit(data_fil[['elbow_to_wrist_cm']])\n    data_fil[['age']] = scalers['scaler_age'].transform(data_fil[['age']])\n    data_fil[['height_cm']] = scalers['scaler_height_cm'].transform(data_fil[['height_cm']])\n    data_fil[['shoulder_to_wrist_cm']] = scalers['scaler_shoulder_to_wrist_cm'].transform(data_fil[['shoulder_to_wrist_cm']])\n    data_fil[['elbow_to_wrist_cm']] = scalers['scaler_elbow_to_wrist_cm'].transform(data_fil[['elbow_to_wrist_cm']])\n    data_fil.drop(columns=['sequence_id', 'subject'], inplace=True)\n    data_tensor = tf.convert_to_tensor(data_fil, dtype=np.float32)\n    return data_tensor, scalers # Shape (Non 7)\n\ntrain_tensor_6, scalers_sub = extract_subject_related_data(train, trainval_demographics, scalers_sub, True)\nval_tensor_6, _ = extract_subject_related_data(val, trainval_demographics, scalers_sub, False)\ntest_tensor_6, _ = extract_subject_related_data(test, test_demographics, scalers_sub, False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:02.228162Z","iopub.execute_input":"2025-07-19T20:42:02.228523Z","iopub.status.idle":"2025-07-19T20:42:02.289039Z","shell.execute_reply.started":"2025-07-19T20:42:02.228498Z","shell.execute_reply":"2025-07-19T20:42:02.288492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract labels\n\ndef extract_labels(data):\n    data_fil = data[['sequence_id', 'gesture']]\n    data_fil = data_fil.drop_duplicates(subset=['sequence_id'])\n    data_fil['gesture'] = data_fil['gesture'].map(class2id)\n    data_fil.drop(columns=['sequence_id'], inplace=True)\n    labels_tensor = tf.convert_to_tensor(data_fil, dtype=np.int32)\n    labels_tensor = tf.squeeze(labels_tensor)\n    return labels_tensor\n\nlabels_train = extract_labels(train)\nlabels_val = extract_labels(val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:02.289772Z","iopub.execute_input":"2025-07-19T20:42:02.290004Z","iopub.status.idle":"2025-07-19T20:42:02.318223Z","shell.execute_reply.started":"2025-07-19T20:42:02.289977Z","shell.execute_reply":"2025-07-19T20:42:02.317667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Explore tensors for shape/min/max/nan values\n\nprint('train_tensor_0 shape is: '+ str(train_tensor_0.shape))\nprint('train_tensor_1 shape is: '+ str(train_tensor_1.shape))\nprint('train_tensor_2 shape is: '+ str(train_tensor_2.shape))\nprint('train_tensor_3 shape is: '+ str(train_tensor_3.shape))\nprint('train_tensor_4 shape is: '+ str(train_tensor_4.shape))\nprint('train_tensor_5 shape is: '+ str(train_tensor_5.shape))\nprint('train_tensor_6 shape is: '+ str(train_tensor_6.shape))\nprint('labels_train shape is: '+ str(labels_train.shape))\n\nfor i in range(0, 7):\n    test_t = globals()[f'train_tensor_{i}']\n    if tf.math.reduce_any(tf.math.is_nan(globals()[f'train_tensor_{i}'])):\n        print(f'train_tensor_{i} is NaN')\n    else:\n        print(f'train_tensor_{i} is NOT NaN')\n    print('Its min value is: '+ str(tf.reduce_min(test_t)))\n    print('Its max value is: '+ str(tf.reduce_max(test_t)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:02.319046Z","iopub.execute_input":"2025-07-19T20:42:02.319798Z","iopub.status.idle":"2025-07-19T20:42:02.368817Z","shell.execute_reply.started":"2025-07-19T20:42:02.319778Z","shell.execute_reply":"2025-07-19T20:42:02.368213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Data augmentation function\n\nmax_roll_fn = 20\ndef translate_tensor(tensors, training=True):\n    tensor_trans_list = []\n    for i in range(train_tensor_0.shape[0]):\n        tensor = tensors[i]\n        if training:\n            roll = random.randint(-max_roll_fn, 0)\n        else:\n            roll = 0\n        tensor_rev = tf.reverse(tensor, axis=[-2])\n        tensor_ext = tf.concat([tensor_rev, tensor, tensor_rev], axis=-2)\n        if tensor_ext.ndim == 3:\n            tensor_trans = tf.roll(tensor_ext, roll, axis=-2)[:, seq_len:2*seq_len, :]\n        else:\n            tensor_trans = tf.roll(tensor_ext, roll, axis=-2)[:, :, seq_len:2*seq_len, :]\n        tensor_trans = tf.expand_dims(tensor_trans, axis=0)\n        tensor_trans_list.append(tensor_trans)\n    tensor_trans = tf.concat(tensor_trans_list, axis=0)\n    return tensor_trans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:02.369454Z","iopub.execute_input":"2025-07-19T20:42:02.369972Z","iopub.status.idle":"2025-07-19T20:42:02.375802Z","shell.execute_reply.started":"2025-07-19T20:42:02.369953Z","shell.execute_reply":"2025-07-19T20:42:02.375126Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Create datasets\n\nSEED=33\nbatch_size=64\nbatch_size_val=64\n\n# Create empty ToF/THM tensors for data augmentation\ntrain_tensor_e = tf.fill(dims=train_tensor_1.shape, value=0.0)\nval_tensor_e = tf.fill(dims=val_tensor_1.shape, value=0.0)\n\n# Create original dataset and same dataset with no ToF/THM sensor values for train and validation\ndataset_train_orig = tf.data.Dataset.from_tensor_slices(({\"input_0\": train_tensor_0, \"input_1\": train_tensor_1,\n                                                          \"input_2\": train_tensor_2, \"input_3\": train_tensor_3,\n                                                          \"input_4\": train_tensor_4, \"input_5\": train_tensor_5,\n                                                          \"input_6\": train_tensor_6}, labels_train))\n\ndataset_train_e = tf.data.Dataset.from_tensor_slices(({\"input_0\": translate_tensor(train_tensor_0), \"input_1\": train_tensor_e,\n                                                       \"input_2\": train_tensor_e, \"input_3\": train_tensor_e,\n                                                       \"input_4\": train_tensor_e, \"input_5\": train_tensor_e,\n                                                       \"input_6\": train_tensor_6}, labels_train))\n\ndataset_train_e2 = tf.data.Dataset.from_tensor_slices(({\"input_0\": translate_tensor(train_tensor_0),\n                                                        \"input_1\": translate_tensor(train_tensor_1),\n                                                        \"input_2\": translate_tensor(train_tensor_2),\n                                                        \"input_3\": translate_tensor(train_tensor_3),\n                                                        \"input_4\": translate_tensor(train_tensor_4),\n                                                        \"input_5\": translate_tensor(train_tensor_5),\n                                                        \"input_6\": train_tensor_6}, labels_train))\ndataset_train_e = dataset_train_e.concatenate(dataset_train_e2)\n\ndataset_val_orig = tf.data.Dataset.from_tensor_slices(({\"input_0\": val_tensor_0, \"input_1\": val_tensor_1,\n                                                        \"input_2\": val_tensor_2, \"input_3\": val_tensor_3,\n                                                        \"input_4\": val_tensor_4, \"input_5\": val_tensor_5,\n                                                        \"input_6\": val_tensor_6}, labels_val))\n\ndataset_val_e = tf.data.Dataset.from_tensor_slices(({\"input_0\": val_tensor_0, \"input_1\": val_tensor_e,\n                                                     \"input_2\": val_tensor_e, \"input_3\": val_tensor_e,\n                                                     \"input_4\": val_tensor_e, \"input_5\": val_tensor_e,\n                                                     \"input_6\": val_tensor_6}, labels_val))\n\n# Create dataset for testing (dummy)\ndataset_test = tf.data.Dataset.from_tensor_slices(({\"input_0\": test_tensor_0, \"input_1\": test_tensor_1,\n                                                    \"input_2\": test_tensor_2, \"input_3\": test_tensor_3,\n                                                    \"input_4\": test_tensor_4, \"input_5\": test_tensor_5,\n                                                    \"input_6\": test_tensor_6}))\ndataset_test = dataset_test.batch(batch_size=1)\n\n# Concatenate original and augmented (wo ToF/THM data) datasets for train and validation\ndataset_train = dataset_train_orig.concatenate(dataset_train_e).shuffle(len(dataset_train_orig)*2,seed=SEED).batch(batch_size=batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\ndataset_val = dataset_val_orig.concatenate(dataset_val_e).batch(batch_size=1).prefetch(tf.data.AUTOTUNE)\ndataset_valbatch = dataset_val_orig.concatenate(dataset_val_e).batch(batch_size=batch_size_val, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n\nprint('Size of train dataset: '+ str(len(dataset_train_orig)*2))\nprint('Steps in train dataset per batch: '+ str(len(dataset_train)) + f' (in {round(len(dataset_train_orig)*2/len(dataset_train))} batches)')\nprint('Size of validation dataset: '+ str(len(dataset_val)))\nprint('Steps in validation dataset per batch: '+ str(len(dataset_valbatch)) + f' (in {round(len(dataset_val)/len(dataset_valbatch))} batches)')\nprint('Size of test dataset: '+ str(len(dataset_test)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:02.376457Z","iopub.execute_input":"2025-07-19T20:42:02.376620Z","iopub.status.idle":"2025-07-19T20:42:17.056216Z","shell.execute_reply.started":"2025-07-19T20:42:02.376605Z","shell.execute_reply":"2025-07-19T20:42:17.055600Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Build and compile neural network","metadata":{}},{"cell_type":"code","source":"## Data augmentation layer for all 2D/3D tensors during training (experimental)\n\nmax_roll_layer = 0\nclass TranslationLayer(tf.keras.layers.Layer):\n    def __init__(self, max_roll, name='augmentation'):\n        super(TranslationLayer, self).__init__(name=name)\n        self.max_roll = max_roll\n    \n    def call(self, tensors, training=True):\n        if training:\n            roll = random.randint(-self.max_roll, 0)\n        else:\n            roll = 0\n        tensors_trans = []\n        for tensor in tensors:\n            tensor_rev = tf.reverse(tensor, axis=[-2])\n            tensor_ext = tf.concat([tensor_rev, tensor, tensor_rev], axis=-2)\n            if tensor.ndim == 5:\n                tensor_trans = tf.roll(tensor_ext, roll, axis=-2)[:, :, :, seq_len:2*seq_len, :]\n            else:\n                tensor_trans = tf.roll(tensor_ext, roll, axis=-2)[:, :, seq_len:2*seq_len, :]\n            tensors_trans.append(tensor_trans)\n        return tuple(tensors_trans)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:17.056865Z","iopub.execute_input":"2025-07-19T20:42:17.057053Z","iopub.status.idle":"2025-07-19T20:42:17.064727Z","shell.execute_reply.started":"2025-07-19T20:42:17.057037Z","shell.execute_reply":"2025-07-19T20:42:17.064060Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Define model branaches for 1D/2D/3D data\n\n# Define 3D CNN model for ToF and THM sensor data\ndef Conv3D_model(input_conv3d, name):\n    reg = L2(1e-4)\n    do = 0.15\n    x1 = tf.keras.layers.Conv3D(filters=64, kernel_size=3, padding='same', kernel_initializer='he_normal',\n                                activation=\"relu\", kernel_regularizer=reg, name=name+'lay1')(input_conv3d)\n    x2 = tf.keras.layers.MaxPool3D(name=name+'mp1')(x1)\n    x3 = tf.keras.layers.BatchNormalization(name=name+'bn1')(x2)\n    x4 = tf.keras.layers.Dropout(do, name=name+'do1')(x3)\n    x5 = tf.keras.layers.Conv3D(filters=128, kernel_size=3, padding='same', kernel_initializer='he_normal',\n                                activation=\"relu\", kernel_regularizer=reg, name=name+'lay2')(x4)\n    x6 = tf.keras.layers.MaxPool3D(name=name+'mp2')(x5)\n    x7 = tf.keras.layers.BatchNormalization(name=name+'bn2')(x6)\n    x8 = tf.keras.layers.Dropout(do, name=name+'do2')(x7)\n    x9 = tf.keras.layers.Conv3D(filters=256, kernel_size=3, padding='same', kernel_initializer='he_normal',\n                                activation=\"relu\", kernel_regularizer=reg, name=name+'lay3')(x8)\n    x10 = tf.keras.layers.MaxPool3D(name=name+'mp3')(x9)\n    x11 = tf.keras.layers.BatchNormalization(name=name+'bn3')(x10)\n    x12 = tf.keras.layers.Dropout(do, name=name+'do3')(x11)\n    x13 = tf.keras.layers.Conv3D(filters=512, kernel_size=(1,1,3), padding='same', kernel_initializer='he_normal',\n                                 activation=\"relu\", kernel_regularizer=reg, name=name+'lay4')(x12)\n    x14 = tf.keras.layers.MaxPool3D(pool_size=(1, 1, 2), name=name+'mp4')(x13)\n    x15 = tf.keras.layers.BatchNormalization(name=name+'bn4')(x14)\n    x16 = tf.keras.layers.Dropout(do, name=name+'do4')(x15)\n    x17 = tf.keras.layers.Conv3D(filters=1024, kernel_size=(1,1,2), padding='valid', kernel_initializer='he_normal',\n                                 activation=\"relu\", kernel_regularizer=reg, name=name+'lay5')(x16)\n    x18 = tf.keras.layers.MaxPool3D(pool_size=(1, 1, 4), name=name+'mp5')(x17)\n    out = tf.keras.layers.GlobalAveragePooling3D(name=name+'gap')(x18)\n    model = tf.keras.Model(inputs=input_conv3d, outputs=out, name=name)\n    return model\n\n# Define 2D CNN model for position and orientation sensor data\ndef Conv2D_model(input_conv2d, name):\n    reg = L2(1e-4)\n    do = 0.15\n    x1 = tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay1')(input_conv2d)\n    x2 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp1')(x1)\n    x3 = tf.keras.layers.BatchNormalization(name=name+'bn1')(x2)\n    x4 = tf.keras.layers.Dropout(do, name=name+'do1')(x3)\n    x5 = tf.keras.layers.Conv2D(filters=128, kernel_size=(3,3), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay2')(x4)\n    x6 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp2')(x5)\n    x7 = tf.keras.layers.BatchNormalization(name=name+'bn2')(x6)\n    x8 = tf.keras.layers.Dropout(do, name=name+'do2')(x7)\n    x9 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3,3), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay3')(x8)\n    x10 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp3')(x9)\n    x11 = tf.keras.layers.BatchNormalization(name=name+'bn3')(x10)\n    x12 = tf.keras.layers.Dropout(do, name=name+'do3')(x11)\n    x13 = tf.keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding='same', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay4')(x12)\n    x14 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp4')(x13)\n    x15 = tf.keras.layers.BatchNormalization(name=name+'bn4')(x14)\n    x16 = tf.keras.layers.Dropout(do, name=name+'do4')(x15)\n    x17 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(3,2), padding='valid', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay5')(x16)\n    x18 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp5')(x17)\n    x19 = tf.keras.layers.BatchNormalization(name=name+'bn5')(x18)\n    x20 = tf.keras.layers.Dropout(do, name=name+'do5')(x19)\n    x21 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(3,2), padding='valid', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay6')(x20)\n    out = tf.keras.layers.GlobalAveragePooling2D(name=name+'gap')(x21)\n    model = tf.keras.Model(inputs=input_conv2d, outputs=out, name=name)\n    return model\n\n# Alternative 2D CNN model for further experiments (optional)\ndef Conv2D_model2(input_conv2d, name):\n    reg = L2(1e-4)\n    do = 0.15\n    x1 = tf.keras.layers.Conv2D(filters=64, kernel_size=(1,10), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay1')(input_conv2d)\n    x2 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp1')(x1)\n    x3 = tf.keras.layers.BatchNormalization(name=name+'bn1')(x2)\n    x4 = tf.keras.layers.Dropout(do, name=name+'do1')(x3)\n    x5 = tf.keras.layers.Conv2D(filters=128, kernel_size=(1,5), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay2')(x4)\n    x6 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp2')(x5)\n    x7 = tf.keras.layers.BatchNormalization(name=name+'bn2')(x6)\n    x8 = tf.keras.layers.Dropout(do, name=name+'do2')(x7)\n    x9 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1,3), padding='same', kernel_initializer='he_normal',\n                                activation='relu', kernel_regularizer=reg, name=name+'lay3')(x8)\n    x10 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp3')(x9)\n    x11 = tf.keras.layers.BatchNormalization(name=name+'bn3')(x10)\n    x12 = tf.keras.layers.Dropout(do, name=name+'do3')(x11)\n    x13 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1,3), padding='same', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay4')(x12)\n    x14 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp4')(x13)\n    x15 = tf.keras.layers.BatchNormalization(name=name+'bn4')(x14)\n    x16 = tf.keras.layers.Dropout(do, name=name+'do4')(x15)\n    x17 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1,2), padding='valid', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay5')(x16)\n    x18 = tf.keras.layers.MaxPool2D(pool_size=(1,2), name=name+'mp5')(x17)\n    x19 = tf.keras.layers.BatchNormalization(name=name+'bn5')(x18)\n    x20 = tf.keras.layers.Dropout(do, name=name+'do5')(x19)\n    x21 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(7,2), padding='valid', kernel_initializer='he_normal',\n                                 activation='relu', kernel_regularizer=reg, name=name+'lay6')(x20)\n    out = tf.keras.layers.GlobalAveragePooling2D(name=name+'gap')(x21)\n    model = tf.keras.Model(inputs=input_conv2d, outputs=out, name=name)\n    return model\n\n# Define 1D DNN model for subject related data\ndef DNN_model(input_dnn, name):\n    reg = L2(1e-4)\n    do = 0.3\n    x1 = tf.keras.layers.Dense(256, activation=\"relu\", kernel_initializer='he_normal', \n                               kernel_regularizer=reg, name=name+'fc1')(input_dnn)\n    x2 = tf.keras.layers.BatchNormalization(name=name+'bn1')(x1)\n    x3 = tf.keras.layers.Dropout(do, name=name+'do1')(x2)\n    x4 = tf.keras.layers.Dense(256, activation=\"relu\", kernel_initializer='he_normal', \n                               kernel_regularizer=reg, name=name+'fc2')(x3)\n    x5 = tf.keras.layers.BatchNormalization(name=name+'bn2')(x4)\n    x6 = tf.keras.layers.Dropout(do, name=name+'do2')(x5)\n    out = tf.keras.layers.Dense(256, activation=\"relu\", kernel_initializer='he_normal', \n                                kernel_regularizer=reg, name=name+'fc3')(x6)\n    model = tf.keras.Model(inputs=input_dnn, outputs=out, name=name)\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:17.066610Z","iopub.execute_input":"2025-07-19T20:42:17.066943Z","iopub.status.idle":"2025-07-19T20:42:17.088190Z","shell.execute_reply.started":"2025-07-19T20:42:17.066926Z","shell.execute_reply":"2025-07-19T20:42:17.087498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Build 1D/2D/3D CNN network\n\ndef build_network():\n    # define the sets of inputs\n    input_0 = tf.keras.Input(shape=(7,seq_len,1), name='input_0')\n    input_1 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_1')\n    input_2 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_2')\n    input_3 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_3')\n    input_4 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_4')\n    input_5 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_5')\n    input_6 = tf.keras.Input(shape=(7,), name='input_6')\n\n    # data augmentation layer (shift time series data along time axis)\n    input_0a, input_1a, input_2a, input_3a, input_4a, input_5a = TranslationLayer(max_roll_layer)(\n        [input_0, input_1, input_2, input_3, input_4, input_5])\n    \n    # define the sets of branches for Conv3D tensors (ToF and THM sensor data)\n    conv3d_model_1_out = Conv3D_model(input_1, name='conv3d_model_1')(input_1a)\n    conv3d_model_2_out = Conv3D_model(input_2, name='conv3d_model_2')(input_2a)\n    conv3d_model_3_out = Conv3D_model(input_3, name='conv3d_model_3')(input_3a)\n    conv3d_model_4_out = Conv3D_model(input_4, name='conv3d_model_4')(input_4a)\n    conv3d_model_5_out = Conv3D_model(input_5, name='conv3d_model_5')(input_5a)\n    \n    # define branch for 2D tensor (IMU sensor data) and for 1D tensor (subject related data)\n    data2d_model_out = Conv2D_model(input_0, name='data2d_model')(input_0a)\n    data1d_model_out = DNN_model(input_6, name='data1d_model')(input_6)\n    \n    # combine the output of the branches\n    combined_3d = tf.keras.layers.Concatenate(name='concat_3d')([conv3d_model_1_out, conv3d_model_2_out,\n                                                                 conv3d_model_3_out, conv3d_model_4_out,\n                                                                 conv3d_model_5_out])\n    combined_3d = tf.keras.layers.BatchNormalization(name='concat_3d_bn')(combined_3d)\n    combined_3d = tf.keras.layers.Dropout(0.8, name='concat_3d_do')(combined_3d)\n    combined_1d_2d = tf.keras.layers.Concatenate(name='concat_1d_2d')([data1d_model_out, data2d_model_out])\n    combined_1d_2d = tf.keras.layers.BatchNormalization(name='concat_1d_2d_bn')(combined_1d_2d)\n    combined_1d_2d = tf.keras.layers.Dropout(0.65, name='concat_1d_2d_do')(combined_1d_2d)\n    combined = tf.keras.layers.Concatenate(name='concat_all')([combined_3d, combined_1d_2d])\n    \n    # apply a FC layer and then a regression prediction on the combined outputs\n    reg = L2(1e-4)\n    do = 0.4\n    z1 = tf.keras.layers.Dense(512, activation=\"relu\", kernel_initializer='he_normal', kernel_regularizer=reg, name='fc1')(combined)\n    z2 = tf.keras.layers.BatchNormalization(name='bn1')(z1)\n    z3 = tf.keras.layers.Dropout(do, name='do1')(z2)\n    z4 = tf.keras.layers.Dense(512, activation=\"relu\", kernel_initializer='he_normal', kernel_regularizer=reg, name='fc2')(z3)\n    z5 = tf.keras.layers.BatchNormalization(name='bn2')(z4)\n    z6 = tf.keras.layers.Dropout(do, name='do2')(z5)\n    z7 = tf.keras.layers.Dense(512, activation=\"relu\", kernel_initializer='he_normal', kernel_regularizer=reg, name='fc3')(z6)\n    z8 = tf.keras.layers.BatchNormalization(name='bn3')(z7)\n    z9 = tf.keras.layers.Dropout(do, name='do3')(z8)\n    z10 = tf.keras.layers.Dense(len(class2bfrb.keys()), activation=\"softmax\", kernel_regularizer=reg, name='prediction')(z9)\n    model = tf.keras.Model(inputs=[input_0, input_1, input_2, input_3, input_4, input_5, input_6], outputs=z10)\n\n    steps_per_epoch = len(dataset_train_orig)*2//batch_size\n    lr_sched = tf.keras.optimizers.schedules.CosineDecayRestarts(5e-4, first_decay_steps=15 * steps_per_epoch)\n    optimizer = tf.keras.optimizers.Adam(0.0005) # SGD(0.001) Adam(0.0001)\n    loss_fn = tf.keras.losses.sparse_categorical_crossentropy #tf.keras.losses.CategoricalCrossentropy()\n    model.compile(optimizer=optimizer, loss=loss_fn, metrics=['accuracy'])\n    return model\n\nwith strategy.scope():\n    model = build_network()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:17.088827Z","iopub.execute_input":"2025-07-19T20:42:17.089039Z","iopub.status.idle":"2025-07-19T20:42:19.539280Z","shell.execute_reply.started":"2025-07-19T20:42:17.089020Z","shell.execute_reply":"2025-07-19T20:42:19.538728Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Explore model architecture\n\nmodel.summary()\ntf.keras.utils.plot_model(model, to_file='model_architecture.png', show_shapes=True, show_dtype=False,\n                          show_layer_names=True, show_layer_activations=True, show_trainable=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:19.539990Z","iopub.execute_input":"2025-07-19T20:42:19.540258Z","iopub.status.idle":"2025-07-19T20:42:20.932914Z","shell.execute_reply.started":"2025-07-19T20:42:19.540229Z","shell.execute_reply":"2025-07-19T20:42:20.932200Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Training","metadata":{}},{"cell_type":"code","source":"## Training parameters\n\nepochs = 200\nTUNING = False\nTRAINING = False\nFINETUNING = False\nsteps_per_epoch = len(dataset_train_orig)*2//batch_size\nvalidation_steps = len(dataset_val)//batch_size_val\n\n## Callback functions\nlr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=8, verbose=1, monitor='val_accuracy')\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=16, verbose=1,\n                                                     monitor='val_accuracy', restore_best_weights=True)\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-5 * 10**(epoch / 20)) # Find starting learning ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:20.933769Z","iopub.execute_input":"2025-07-19T20:42:20.934084Z","iopub.status.idle":"2025-07-19T20:42:20.940893Z","shell.execute_reply.started":"2025-07-19T20:42:20.934059Z","shell.execute_reply":"2025-07-19T20:42:20.940198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Load pre-trained weights\n\nif not TRAINING and not TUNING:\n    model.load_weights('/kaggle/input/cmi-422/CMI_5_6_19.weights.h5')\n    print('Model weights have been loaded!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:20.941529Z","iopub.execute_input":"2025-07-19T20:42:20.941755Z","iopub.status.idle":"2025-07-19T20:42:20.956369Z","shell.execute_reply.started":"2025-07-19T20:42:20.941730Z","shell.execute_reply":"2025-07-19T20:42:20.955753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train model\n\nif TRAINING or FINETUNING:\n    history = model.fit(dataset_train, validation_data=dataset_valbatch, epochs=epochs,\n                        callbacks=[lr_scheduler, early_stopping_cb])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:20.957315Z","iopub.execute_input":"2025-07-19T20:42:20.957546Z","iopub.status.idle":"2025-07-19T20:42:20.970635Z","shell.execute_reply.started":"2025-07-19T20:42:20.957531Z","shell.execute_reply":"2025-07-19T20:42:20.970133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Tuner\n\nif TUNING:\n    i_TunerTyp = 1 # Choose desired tuner type: {1: 'grid', 2: 'random', 3: 'hyper'}\n    TunerStr = {1: 'grid', 2: 'random', 3: 'hyper'}\n\n    tuner_grid = kt.GridSearch(hypermodel=build_network, objective='val_accuracy',\n                               max_trials=10, max_consecutive_failed_trials=1,\n                               overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\n    tuner_random = kt.RandomSearch(hypermodel=build_network, objective='val_accuracy',\n                                   max_trials=10, executions_per_trial=1,\n                                   overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\n    tuner_hyper = kt.Hyperband(hypermodel=build_network, objective='val_accuracy',\n                               max_epochs=90, factor=5, hyperband_iterations=1,\n                               overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\n    tuner = globals()[f'tuner_{TunerStr[i_TunerTyp]}']\n    tuner.search_space_summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:20.971091Z","iopub.execute_input":"2025-07-19T20:42:20.971250Z","iopub.status.idle":"2025-07-19T20:42:22.559982Z","shell.execute_reply.started":"2025-07-19T20:42:20.971236Z","shell.execute_reply":"2025-07-19T20:42:22.559217Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train model by Tuner\n\nif TUNING:\n    tuner.search(dataset_train, validation_data=dataset_valbatch, epochs=epochs,\n                     callbacks=[lr_scheduler, early_stopping_cb])\n\n    best_models = tuner.get_best_models(num_models=2)\n    model = best_models[0]\n    model.summary()\n    tuner.results_summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T20:42:22.560771Z","iopub.execute_input":"2025-07-19T20:42:22.561007Z","iopub.status.idle":"2025-07-19T20:50:45.260919Z","shell.execute_reply.started":"2025-07-19T20:42:22.560981Z","shell.execute_reply":"2025-07-19T20:50:45.259933Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Save weights of model\n\nif TRAINING or TUNING or FINETUNING:\n    model.save_weights('CMI_5_9_33.weights.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T13:11:53.603695Z","iopub.execute_input":"2025-07-16T13:11:53.604065Z","iopub.status.idle":"2025-07-16T13:11:55.903091Z","shell.execute_reply.started":"2025-07-16T13:11:53.604034Z","shell.execute_reply":"2025-07-16T13:11:55.897049Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Evaluation","metadata":{}},{"cell_type":"code","source":"## Plot learning curves\n\nif TRAINING or FINETUNING:\n    history_fil = {key: history.history[key] for key in ['accuracy', 'val_accuracy']}\n    history_fil2 = {key: history.history[key] for key in ['loss', 'val_loss']}\n    history_fil3 = {key: history.history[key] for key in ['learning_rate']}\n    \n    pd.DataFrame(history_fil).plot()\n    plt.ylabel(\"Accuracy\")\n    plt.xlabel(\"epochs\")\n    pd.DataFrame(history_fil2).plot()\n    plt.ylabel(\"Loss\")\n    plt.xlabel(\"epochs\")\n    plt.axis([10, len(history_fil2['val_loss']), 0, history_fil2['val_loss'][10]+0.1*history_fil2['val_loss'][10]])\n    pd.DataFrame(history_fil3).plot()\n    plt.ylabel(\"Learning rate\")\n    plt.xlabel(\"epochs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T15:44:27.029399Z","iopub.execute_input":"2025-07-16T15:44:27.029781Z","iopub.status.idle":"2025-07-16T15:44:27.549053Z","shell.execute_reply.started":"2025-07-16T15:44:27.029750Z","shell.execute_reply":"2025-07-16T15:44:27.544200Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Calculate competition specific score\n\nfrom cmi_2025_metric_copy_for_import import CompetitionMetric\n\n# Get predicted labels for the validation set\nimages, labels = tuple(zip(*dataset_val))\n\nprint(model.evaluate(dataset_val))\nprobabilities = model.predict(dataset_val)\ny_val_pred = np.argmax(probabilities, axis=-1)\ny_val_true = np.squeeze(np.array(labels))\n\n# Map integer labels back to gesture strings\nval_pred_labels = pd.Series(y_val_pred).map(lambda i: id2class[i])\nval_true_labels = pd.Series(y_val_true).map(lambda i: id2class[i])\n\n# Build DataFrames for the metric\nval_submission = pd.DataFrame({'gesture': val_pred_labels})\nval_solution = pd.DataFrame({'gesture': val_true_labels})\n\n# Run competition metric\nmetric = CompetitionMetric()\nscore = metric.calculate_hierarchical_f1(val_solution, val_submission)\nprint(f\"Estimated leaderboard (val) score: {score:.4f}\")\n\nprint(val_pred_labels)\nprint(val_true_labels)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-16T13:09:07.551199Z","iopub.execute_input":"2025-07-16T13:09:07.551581Z","iopub.status.idle":"2025-07-16T13:11:10.904297Z","shell.execute_reply.started":"2025-07-16T13:09:07.551552Z","shell.execute_reply":"2025-07-16T13:11:10.899680Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Submission","metadata":{}},{"cell_type":"code","source":"## Init server\n\nimport kaggle_evaluation.cmi_inference_server\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    test = sequence.to_pandas()\n    if len(test.index) > seq_len:\n        print(len(test.index))\n        test = test[-seq_len:]\n    #print(len(test.index))\n\n    test_demographics = demographics.to_pandas()\n    test_tensor_0, test_tensor_1, test_tensor_2, test_tensor_3, test_tensor_4, test_tensor_5 = extract_time_series_data(test, False)\n    test_tensor_6, _ = extract_subject_related_data(test, test_demographics, scalers_sub, False)\n    dataset_test = tf.data.Dataset.from_tensor_slices(({\"input_0\": test_tensor_0,\n                                                        \"input_1\": test_tensor_1, \"input_2\": test_tensor_2,\n                                                        \"input_3\": test_tensor_3, \"input_4\": test_tensor_4,\n                                                        \"input_5\": test_tensor_5, \"input_6\": test_tensor_6}))\n    dataset_test = dataset_test.batch(batch_size=1)\n    \n    probabilities = model.predict(dataset_test, verbose=0)\n    predictions = np.argmax(probabilities, axis=-1)\n    prediction = np.argmax(probabilities, axis=-1)[0]\n    return id2class[prediction]\n\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Run server\n\nif not TRAINING and not TUNING and not FINETUNING:\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(\n            data_paths=(\n                '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n                '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n            )\n        )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Experimenting code","metadata":{}},{"cell_type":"code","source":"# ## Explore max/min/average length of sequences\n# data_check = train[train['phase']=='Gesture'] # Max: 71\n# data_check = train # Max: 700, Min: 29, Avg: 71 ('SEQ_015261')\n# min_length = 1000\n# max_length = 0\n# for s, sequence_id in enumerate(data_check['sequence_id'].unique()):\n#     if s%1000 == 0: print(s) # Show progress\n#     data_check_fil = data_check[data_check['sequence_id'] == sequence_id]\n#     seq_length = len(data_check_fil.index)\n#     if seq_length < min_length:\n#         min_length = seq_length\n#         min_seq_id = sequence_id\n#     if seq_length > max_length:\n#         max_length = seq_length\n#         max_seq_id = sequence_id","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ## Explore split distribution comparing Dataset.shuffle with StratifiedGroupKFold\n# SEED = 33\n# train_split = 0.9\n# labels = extract_labels(trainval)\n# dataset_label = tf.data.Dataset.from_tensor_slices(labels)\n# dataset_label_train = dataset_label.shuffle(8200,seed=SEED).take(round(len(labels)*train_split))\n# dataset_label_val = dataset_label.shuffle(8200,seed=SEED).skip(round(len(labels)*train_split))\n\n# labels_train = extract_labels(train)\n# labels_val = extract_labels(val)\n# fold_label_train = tf.data.Dataset.from_tensor_slices(labels_train)\n# fold_label_val = tf.data.Dataset.from_tensor_slices(labels_val)\n\n# def count_classes(labels):\n#     label_reconstructed = []\n#     for i, label in enumerate(labels):\n#         label_reconstructed.append(label)\n#     label_reconstructed = tf.convert_to_tensor(label_reconstructed, dtype=np.int32)\n#     y, idx, count = tf.unique_with_counts(label_reconstructed)\n#     count = count/len(label_reconstructed)\n#     return y, idx, count\n\n# y, idx, count = count_classes(dataset_label)\n# distribution = pd.DataFrame(data = count, index = y, columns=['Orig'])\n# y, idx, count = count_classes(dataset_label_train)\n# distribution = distribution.merge(pd.DataFrame(data = count, index = y, columns=['DS_Train']), how='left', left_index=True, right_index=True)\n# y, idx, count = count_classes(dataset_label_val)\n# distribution = distribution.merge(pd.DataFrame(data = count, index = y, columns=['DS_Val']), how='left', left_index=True, right_index=True)\n# y, idx, count = count_classes(fold_label_train)\n# distribution = distribution.merge(pd.DataFrame(data = count, index = y, columns=['Fold_Train']), how='left', left_index=True, right_index=True)\n# y, idx, count = count_classes(fold_label_val)\n# distribution = distribution.merge(pd.DataFrame(data = count, index = y, columns=['Fold_Val']), how='left', left_index=True, right_index=True)\n# distribution","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ## Plot learning curves for definition of start leraning rate\n# lrs = 1e-5 * (10 ** (np.arange(100) / 20)) # Define the learning rate array\n# plt.figure(figsize=(10, 6)) # Set the figure size\n# plt.grid(True) # Set the grid\n# plt.semilogx(lrs, history.history[\"loss\"]) # Plot the loss in log scale\n# plt.tick_params('both', length=10, width=1, which='both') # Increase the tickmarks size\n# plt.axis([1e-5, 1e-0, 0, 10]) # Set the plot boundaries","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Draft backup code (for debuging purposes)","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n# import matplotlib.pyplot as plt\n# from mpl_toolkits.mplot3d import Axes3D\n# from ipywidgets import interact\n# #%matplotlib inline\n\n# def interactive_plot(image3d):\n#     # only for example, use your grid\n#     z = np.linspace(0, 8, 8)\n#     x = np.linspace(0, 8, 8)\n#     y = np.linspace(0, 8, 8)\n    \n#     X, Y, Z = np.meshgrid(x, y, z)\n    \n#     # Your 4dimension, only for example (use yours)\n#     U = image3d_tof_1[0].numpy().reshape(8, 8, 100)\n#     U = U[:, :, :8]\n#     # Creating figure\n#     fig = plt.figure(figsize=(8,8))\n#     ax = fig.add_subplot(111, projection='3d')\n#     ax.view_init(elev=-84, azim=90, roll=0)\n#     #ax = Axes3D(fig)\n#     #ax = plt.axes(projection=\"3d\")\n    \n#     # Creating plot\n#     scatter = ax.scatter3D(X, Y, Z, c=U, s=200, marker='o', alpha=0.5, cmap='PRGn')\n#     #fig.colorbar(scatter, ax=ax)\n#     plt.show()\n\n# interact(interactive_plot(image3d_tof_1[0]))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # Explore sequence order of different grouping strategies\n# seq_gp = train.groupby('sequence_id')\n# groups = np.array([seq_id for seq_id, _ in seq_gp])\n\n# groups2 = train['sequence_id'].unique()\n\n# train_fil = train[['sequence_id', 'subject']].drop_duplicates(subset=['sequence_id'])\n# train_fil = train_fil.merge(trainval_demographics, how='left', left_on='subject', right_on='subject')\n# groups3 = train_fil['sequence_id'].to_numpy()\n\n# print(groups)\n# print(groups2)\n# print(groups3)\n# if np.array_equal(groups, groups2) and np.array_equal(groups, groups3):\n#     print(\"Equal\")\n# else:\n#     print(\"Not Equal\")","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}