{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12611603,"sourceType":"datasetVersion","datasetId":7699074},{"sourceId":12934262,"sourceType":"datasetVersion","datasetId":7906263},{"sourceId":242954653,"sourceType":"kernelVersion"}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Setup environment (CPU/GPU/TPU) and import packages","metadata":{}},{"cell_type":"code","source":"## For TPU environment (install missing packages / reinstall tensorflow to solve NaN topic during training / restart kernel)\n\nimport IPython\nimport tensorflow as tf\n\nif len(tf.config.experimental.list_logical_devices('TPU'))>0:\n    !pip install -q tensorflow-tpu -f https://storage.googleapis.com/libtpu-tf-releases/index.html --force-reinstall\n    !pip install -q pydot\n    !pip install -q -U keras-tuner\n    !pip install xgboost\n    IPython.Application.instance().kernel.do_shutdown(True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:13:25.643452Z","iopub.execute_input":"2025-09-01T13:13:25.643680Z","iopub.status.idle":"2025-09-01T13:13:25.647849Z","shell.execute_reply.started":"2025-09-01T13:13:25.643663Z","shell.execute_reply":"2025-09-01T13:13:25.647086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Import packages\n\nimport os\nimport time\nimport polars as pl\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nimport keras_tuner as kt\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import imshow\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom tensorflow.keras.regularizers import L2\nprint('Tensorflow version: ' + tf.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:56:51.829223Z","iopub.execute_input":"2025-09-01T13:56:51.829765Z","iopub.status.idle":"2025-09-01T13:57:06.670105Z","shell.execute_reply.started":"2025-09-01T13:56:51.829742Z","shell.execute_reply":"2025-09-01T13:57:06.669403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Detect hardware (CPU/GPU/TPU), setup environment and return appropriate distribution strategy\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect(tpu='local') # set tpu is local as it should be available in the VM\n    print('✅ Running on TPU ', tpu.master())\nexcept:\n    print('❌ Using CPU/GPU')\n    tpu = None\n\nif tpu:\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy() # default distribution strategy in Tensorflow. Works on CPU and single GPU.\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:57:06.671333Z","iopub.execute_input":"2025-09-01T13:57:06.671840Z","iopub.status.idle":"2025-09-01T13:57:07.708915Z","shell.execute_reply.started":"2025-09-01T13:57:06.671820Z","shell.execute_reply":"2025-09-01T13:57:07.708122Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Load and preprocess data into datasets","metadata":{}},{"cell_type":"code","source":"## Read csv files\n\npath = '/kaggle/input/cmi-detect-behavior-with-sensor-data/'\ntrainval = pd.read_csv(path + \"train.csv\")\ntrainval_demographics = pd.read_csv(path + \"train_demographics.csv\")\ntest = pd.read_csv(path + \"test.csv\")\ntest_demographics = pd.read_csv(path + \"test_demographics.csv\")\n#trainval = trainval[:50000] # To save time during debugging/expermenting","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:57:07.709718Z","iopub.execute_input":"2025-09-01T13:57:07.710436Z","iopub.status.idle":"2025-09-01T13:57:37.237912Z","shell.execute_reply.started":"2025-09-01T13:57:07.710412Z","shell.execute_reply":"2025-09-01T13:57:37.237294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Spliting the train data into train and val\n\ncv = StratifiedGroupKFold(n_splits=10)\nfor fold, (train_idx, val_idx) in enumerate(cv.split(trainval, trainval['gesture'],\n                                                     groups=np.array(trainval['sequence_id'], dtype='<U10'))):\n    print(train_idx)\n    print(val_idx)\n    train, val = trainval.iloc[train_idx], trainval.iloc[val_idx]\n    print(f\"✅ Fold {fold}: Train size = {len(train_idx)}, Val size = {len(val_idx)}\")\n    break  # Use only the first fold for now","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:57:37.239412Z","iopub.execute_input":"2025-09-01T13:57:37.239655Z","iopub.status.idle":"2025-09-01T13:57:44.437428Z","shell.execute_reply.started":"2025-09-01T13:57:37.239637Z","shell.execute_reply":"2025-09-01T13:57:44.436655Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Generate transformation dictionaries for classes\n\nclass2bfrb = {'Cheek - pinch skin': 1, 'Forehead - pull hairline': 1, 'Write name on leg': 0,\n              'Feel around in tray and pull out an object': 0, 'Neck - scratch': 1,\n              'Neck - pinch skin': 1, 'Eyelash - pull hair': 1, 'Eyebrow - pull hair': 1,\n              'Forehead - scratch': 1, 'Above ear - pull hair': 1, 'Wave hello': 0, 'Write name in air': 0,\n              'Text on phone': 0, 'Pull air toward your face': 0, 'Pinch knee/leg skin': 0,\n              'Scratch knee/leg skin': 0, 'Drink from bottle/cup': 0, 'Glasses on/off': 0}\nclass2id = {key: i for i, key in enumerate(class2bfrb.keys())}\nid2class = {i: value for i, value in enumerate(class2bfrb.keys())}\nid2bfrb = {i: value for i, value in enumerate(class2bfrb.values())}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:57:44.438185Z","iopub.execute_input":"2025-09-01T13:57:44.438376Z","iopub.status.idle":"2025-09-01T13:57:44.443624Z","shell.execute_reply.started":"2025-09-01T13:57:44.438360Z","shell.execute_reply":"2025-09-01T13:57:44.442922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract sensor related data\n\nscalers = {'scaler_acc' : MinMaxScaler().fit([[-50],[50]]), 'scaler_rot' : MinMaxScaler().fit([[-1],[1]]),\n           'scaler_tof' : MinMaxScaler().fit([[-1],[249]]), 'scaler_thm' : MinMaxScaler().fit([[-1],[40]])}\nseq_len=80\n\ndef extract_time_series_data(data, training):\n    start_time = time.time()\n    poss_tensor_padded_list = []\n    for i in range(1, 6):\n       globals()[f'tof_thm_tensor_padded_list{i}'] = []\n    \n    for s, sequence_id in enumerate(data['sequence_id'].unique()):\n        if s%1000 == 0: # Show progress\n            start_time2 = time.time()\n        \n        # Filter data for one sequence and truncate if longer than seq_len\n        data_seq = data[data['sequence_id'] == sequence_id].ffill().bfill().fillna(0)\n        if len(data_seq.index) > seq_len:\n            data_seq = data_seq[-seq_len:]\n            \n        # Extract position/orientation related data\n        pos_tensor_padded_list = []\n        for column in ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']:\n            pos_array  = pd.DataFrame(data_seq, columns=[column]).to_numpy()\n            if column in ['acc_x', 'acc_y', 'acc_z']:\n                pos_array = scalers['scaler_acc'].transform(pos_array)\n            pos_tensor = tf.convert_to_tensor(pos_array.reshape(1,1,-1,1), dtype=tf.float32)\n            pad_length = seq_len - pos_tensor.shape[2] # Max length in train dataset is 71 min length is 3 after removing transitions\n            paddings_2d = tf.constant([[0, 0,], [0, 0], [pad_length, 0], [0, 0]])\n            pos_tensor_padded = tf.pad(pos_tensor, paddings_2d, \"CONSTANT\")  # Shape (1,1,seq_len,1)\n            pos_tensor_padded_list.append(pos_tensor_padded)\n        poss_tensor_padded = tf.concat(pos_tensor_padded_list, axis=1) # Shape (1,7,seq_len,1)\n        poss_tensor_padded_list.append(poss_tensor_padded)\n\n        # Extract ToF/THM related data\n        for i in range(1, 6):\n            # Extract ToF data\n            image_arrays  = pd.DataFrame(data_seq, columns=[f'tof_{i}_v{j}' for j in range(64)]).to_numpy()\n            image_arrays = (image_arrays + 1)/250\n            image3d_tensor = tf.convert_to_tensor(image_arrays.reshape(1,8,8,-1,1), dtype=tf.float32)\n            paddings_3d = tf.constant([[0, 0,], [0, 0], [0, 0], [pad_length, 0], [0, 0]])\n            image3d_tensor_padded = tf.pad(image3d_tensor, paddings_3d, \"CONSTANT\")  # Shape (1,8,8,seq_len,1)\n            # Extract THM data\n            temp_array  = pd.DataFrame(data_seq, columns=[f'thm_{i}']).to_numpy()\n            temp_array = scalers['scaler_thm'].transform(temp_array)\n            temp_tensor = tf.convert_to_tensor(temp_array.reshape(1,1,1,-1,1), dtype=tf.float32)\n            multiply_factor = tf.constant([1,8,8,1,1], tf.int32)\n            temp_tensor = tf.tile(temp_tensor, multiply_factor)\n            temp_tensor_padded = tf.pad(temp_tensor, paddings_3d, \"CONSTANT\") # Shape (1,8,8,seq_len,1)\n            tof_thm_tensor_padded = tf.concat([image3d_tensor_padded, temp_tensor_padded], axis=4) # Shape (1,8,8,seq_len,2)\n            globals()[f'tof_thm_tensor_padded_list{i}'].append(tof_thm_tensor_padded)\n        if s%1000 == 0: print(\"Time taken for 1000 seqs: %.2fs\" % (time.time() - start_time2))\n    \n    data_tensor_0 = tf.concat(poss_tensor_padded_list, axis=0)\n    for i in range(1, 6):\n        globals()[f'data_tensor_{i}'] = tf.concat(globals()[f'tof_thm_tensor_padded_list{i}'], axis=0)\n    print(\"Time taken: %.2fs\" % (time.time() - start_time))\n    return data_tensor_0, data_tensor_1, data_tensor_2, data_tensor_3, data_tensor_4, data_tensor_5\n\ntrain_tensor_0, train_tensor_1, train_tensor_2, train_tensor_3, train_tensor_4, train_tensor_5 = extract_time_series_data(train, True)\nval_tensor_0, val_tensor_1, val_tensor_2, val_tensor_3, val_tensor_4, val_tensor_5 = extract_time_series_data(val, False)\ntest_tensor_0, test_tensor_1, test_tensor_2, test_tensor_3, test_tensor_4, test_tensor_5 = extract_time_series_data(test, False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T13:57:44.444420Z","iopub.execute_input":"2025-09-01T13:57:44.444623Z","iopub.status.idle":"2025-09-01T14:04:28.861473Z","shell.execute_reply.started":"2025-09-01T13:57:44.444608Z","shell.execute_reply":"2025-09-01T14:04:28.860718Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract Subject related data\n\nscalers_sub = {'scaler_age' : StandardScaler(), 'scaler_height_cm' : StandardScaler(),\n               'scaler_shoulder_to_wrist_cm' : StandardScaler(), 'scaler_elbow_to_wrist_cm' : StandardScaler()}\n\ndef extract_subject_related_data(data, data_demographics, scalers, training):\n    # Filter subject related data that every sequence id apperas only once\n    data_fil = data[['sequence_id', 'subject']].drop_duplicates(subset=['sequence_id'])\n    data_fil = data_fil.merge(data_demographics, how='left', left_on='subject', right_on='subject')\n    \n    # Scaling numeric values\n    data_fil[['adult_child', 'sex', 'handedness']] = data_fil[['adult_child', 'sex', 'handedness']].astype(dtype=np.float32)\n    if training:\n        scalers['scaler_age'].fit(data_fil[['age']])\n        scalers['scaler_height_cm'].fit(data_fil[['height_cm']])\n        scalers['scaler_shoulder_to_wrist_cm'].fit(data_fil[['shoulder_to_wrist_cm']])\n        scalers['scaler_elbow_to_wrist_cm'].fit(data_fil[['elbow_to_wrist_cm']])\n    data_fil[['age']] = scalers['scaler_age'].transform(data_fil[['age']])\n    data_fil[['height_cm']] = scalers['scaler_height_cm'].transform(data_fil[['height_cm']])\n    data_fil[['shoulder_to_wrist_cm']] = scalers['scaler_shoulder_to_wrist_cm'].transform(data_fil[['shoulder_to_wrist_cm']])\n    data_fil[['elbow_to_wrist_cm']] = scalers['scaler_elbow_to_wrist_cm'].transform(data_fil[['elbow_to_wrist_cm']])\n    data_fil.drop(columns=['sequence_id', 'subject'], inplace=True)\n    data_tensor = tf.convert_to_tensor(data_fil, dtype=np.float32)\n    return data_tensor, scalers # Shape (Non 7)\n\ntrain_tensor_6, scalers_sub = extract_subject_related_data(train, trainval_demographics, scalers_sub, True)\nval_tensor_6, _ = extract_subject_related_data(val, trainval_demographics, scalers_sub, False)\ntest_tensor_6, _ = extract_subject_related_data(test, test_demographics, scalers_sub, False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:28.862343Z","iopub.execute_input":"2025-09-01T14:04:28.862691Z","iopub.status.idle":"2025-09-01T14:04:28.933127Z","shell.execute_reply.started":"2025-09-01T14:04:28.862672Z","shell.execute_reply":"2025-09-01T14:04:28.932506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Extract labels\n\ndef extract_labels(data):\n    data_fil = data[['sequence_id', 'gesture']]\n    data_fil = data_fil.drop_duplicates(subset=['sequence_id'])\n    data_fil['gesture'] = data_fil['gesture'].map(class2id)\n    data_fil.drop(columns=['sequence_id'], inplace=True)\n    labels_tensor = tf.convert_to_tensor(data_fil, dtype=np.int32)\n    labels_tensor = tf.squeeze(labels_tensor)\n    return labels_tensor\n\nlabels_train = extract_labels(train)\nlabels_val = extract_labels(val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:28.933840Z","iopub.execute_input":"2025-09-01T14:04:28.934021Z","iopub.status.idle":"2025-09-01T14:04:28.968810Z","shell.execute_reply.started":"2025-09-01T14:04:28.934005Z","shell.execute_reply":"2025-09-01T14:04:28.968205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Explore tensors for shape/min/max/nan values\n\nprint('train_tensor_0 shape is: '+ str(train_tensor_0.shape))\nprint('train_tensor_1 shape is: '+ str(train_tensor_1.shape))\nprint('train_tensor_2 shape is: '+ str(train_tensor_2.shape))\nprint('train_tensor_3 shape is: '+ str(train_tensor_3.shape))\nprint('train_tensor_4 shape is: '+ str(train_tensor_4.shape))\nprint('train_tensor_5 shape is: '+ str(train_tensor_5.shape))\nprint('train_tensor_6 shape is: '+ str(train_tensor_6.shape))\nprint('labels_train shape is: '+ str(labels_train.shape))\n\nfor i in range(0, 7):\n    test_t = globals()[f'train_tensor_{i}']\n    if tf.math.reduce_any(tf.math.is_nan(globals()[f'train_tensor_{i}'])):\n        print(f'train_tensor_{i} is NaN')\n    else:\n        print(f'train_tensor_{i} is NOT NaN')\n    print('Its min value is: '+ str(tf.reduce_min(test_t)))\n    print('Its max value is: '+ str(tf.reduce_max(test_t)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:28.969541Z","iopub.execute_input":"2025-09-01T14:04:28.969752Z","iopub.status.idle":"2025-09-01T14:04:29.025363Z","shell.execute_reply.started":"2025-09-01T14:04:28.969735Z","shell.execute_reply":"2025-09-01T14:04:29.024731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# scale = 2\n# data = tf.clip_by_value(tf.cast(tf.random.normal(shape=[10000], mean=80,  stddev=10*scale, dtype=tf.float32), dtype=tf.int64),\n#                  round(10), round(80*scale)).numpy()\n# plt.hist(data, bins=150)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-31T18:22:56.266462Z","iopub.execute_input":"2025-08-31T18:22:56.266634Z","iopub.status.idle":"2025-08-31T18:22:56.269639Z","shell.execute_reply.started":"2025-08-31T18:22:56.266620Z","shell.execute_reply":"2025-08-31T18:22:56.269052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Data augmentation layer for all 2D/3D tensors during training or as pre-processing\n\nbuild_in_augmentation = True\n\nclass TranslationLayer(tf.keras.layers.Layer):\n    def __init__(self, max_roll, max_acc_scale, max_t_scale, name='augmentation'):\n        super(TranslationLayer, self).__init__(name=name)\n        self.max_roll = max_roll\n        self.max_acc_scale = max_acc_scale\n        self.max_t_scale = max_t_scale\n        #with tf.device('/GPU:0'): # Workaround to solve error in GPU environment\n        self.roll = tf.Variable(0, name='roll', dtype=tf.int64, trainable=False)\n        self.scale = tf.Variable(1.0, name='scale', dtype=tf.float32, trainable=False)\n        self.t_scale = tf.Variable(80, name='t_scale', dtype=tf.int64, trainable=False)\n    \n    def call(self, tensors, training=None):\n        if training:\n            if self.max_roll > 0:\n                self.roll.assign(tf.random.uniform(shape=(), minval=-self.max_roll, maxval=0, dtype=tf.int64))\n            self.scale.assign(tf.random.uniform(shape=(), minval=1/self.max_acc_scale, maxval=self.max_acc_scale, dtype=tf.float32))\n            self.t_scale.assign(tf.random.uniform(shape=(), minval=round(80/self.max_t_scale),\n                                                  maxval=round(80*self.max_t_scale), dtype=tf.int64))\n            # self.t_scale.assign(tf.clip_by_value(tf.cast(\n            #     tf.random.normal(shape=(), mean=80,  stddev=10*self.max_t_scale, dtype=tf.float32), dtype=tf.int64),\n            #                                      round(40/self.max_t_scale), round(80*self.max_t_scale)))\n            pass\n        else:\n            self.roll.assign(0)\n            self.scale.assign(1)\n            self.t_scale.assign(80)\n            pass\n        tensors_trans = []\n        for tensor in tensors:\n            tensor_rev = tf.reverse(tensor, axis=[-2])\n            tensor_ext = tf.concat([tensor_rev, tensor, tensor_rev, tensor, tensor_rev], axis=-2)\n            if tensor.ndim == 5:\n                tensor_trans = tf.roll(tensor_ext, self.roll, axis=-2)\n                tensor_trans = tf.reshape(tensor_trans, (-1, 64, 400, 2))[:, :, -(self.t_scale+seq_len):-seq_len, :]\n                tensor_trans = tf.image.resize(tensor_trans, [64, 80])\n                tensor_trans = tf.reshape(tensor_trans, (-1, 8, 8, 80, 2))\n            else:\n\n                tensor_trans = tf.roll(tensor_ext, self.roll, axis=-2)\n                tensor_trans = tensor_trans[:, :, -(self.t_scale+seq_len):-seq_len, :]\n                tensor_trans = tf.image.resize(tensor_trans, [7, 80])              \n                #tensor_trans_acc, tensor_trans_rot = tf.split(tensor_trans, num_or_size_splits=[3,4], axis=1)\n                #tensor_trans_acc = tensor_trans_acc*self.scale\n                #tensor_trans = tf.concat([tensor_trans_acc, tensor_trans_rot], axis=1)\n            tensors_trans.append(tensor_trans)\n        #tf.print(self.roll)\n        #tf.print(self.scale)\n        #tf.print(self.t_scale)\n        return tuple(tensors_trans)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:29.027226Z","iopub.execute_input":"2025-09-01T14:04:29.027456Z","iopub.status.idle":"2025-09-01T14:04:29.037535Z","shell.execute_reply.started":"2025-09-01T14:04:29.027438Z","shell.execute_reply":"2025-09-01T14:04:29.036881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Data augmentation as pre-processing for experimental purposes (image.resize is not working on GPU/TPU fit or tuner))\n\nif not build_in_augmentation:\n    max_roll_layer = 0\n    max_acc_scale = 1\n    max_t_scale = 2\n    train_tensor_00_aug_list = []\n    train_tensor_0_aug_list = []\n    train_tensor_1_aug_list = []\n    train_tensor_2_aug_list = []\n    train_tensor_3_aug_list = []\n    train_tensor_4_aug_list = []\n    train_tensor_5_aug_list = []\n    for i in range(train_tensor_0.shape[0]):\n        input_00a = TranslationLayer(max_roll_layer, max_acc_scale, max_t_scale)(\n            [train_tensor_0[i:i+1]], training=True)\n        input_0a, input_1a, input_2a, input_3a, input_4a, input_5a = TranslationLayer(max_roll_layer, max_acc_scale, max_t_scale)(\n            [train_tensor_0[i:i+1], train_tensor_1[i:i+1], train_tensor_2[i:i+1],\n             train_tensor_3[i:i+1], train_tensor_4[i:i+1], train_tensor_5[i:i+1]], training=True)\n        train_tensor_00_aug_list.append(input_00a)\n        train_tensor_0_aug_list.append(input_0a)\n        train_tensor_1_aug_list.append(input_1a)\n        train_tensor_2_aug_list.append(input_2a)\n        train_tensor_3_aug_list.append(input_3a)\n        train_tensor_4_aug_list.append(input_4a)\n        train_tensor_5_aug_list.append(input_5a)\n    train_tensor_00_aug = tf.concat(train_tensor_00_aug_list, axis=0)\n    train_tensor_00_aug = tf.squeeze(train_tensor_00_aug, axis=[1])\n    train_tensor_0_aug = tf.concat(train_tensor_0_aug_list, axis=0)\n    train_tensor_1_aug = tf.concat(train_tensor_1_aug_list, axis=0)\n    train_tensor_2_aug = tf.concat(train_tensor_2_aug_list, axis=0)\n    train_tensor_3_aug = tf.concat(train_tensor_3_aug_list, axis=0)\n    train_tensor_4_aug = tf.concat(train_tensor_4_aug_list, axis=0)\n    train_tensor_5_aug = tf.concat(train_tensor_5_aug_list, axis=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:29.038094Z","iopub.execute_input":"2025-09-01T14:04:29.038288Z","iopub.status.idle":"2025-09-01T14:04:29.052069Z","shell.execute_reply.started":"2025-09-01T14:04:29.038273Z","shell.execute_reply":"2025-09-01T14:04:29.051445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Create datasets with original data and only IMU data (as in hidden test data)\n\nSEED=33\nbatch_size=64\nbatch_size_val=64\n\n# Create empty ToF/THM tensors for data augmentation\ntrain_tensor_e = tf.fill(dims=train_tensor_1.shape, value=0.0)\nval_tensor_e = tf.fill(dims=val_tensor_1.shape, value=0.0)\n\n# Create original dataset and same dataset with no ToF/THM sensor values for train and validation\ndataset_train_orig = tf.data.Dataset.from_tensor_slices(({\"input_0\": train_tensor_0, \"input_1\": train_tensor_1,\n                                                          \"input_2\": train_tensor_2, \"input_3\": train_tensor_3,\n                                                          \"input_4\": train_tensor_4, \"input_5\": train_tensor_5,\n                                                          \"input_6\": train_tensor_6}, labels_train))\n\ndataset_train_e = tf.data.Dataset.from_tensor_slices(({\"input_0\": train_tensor_0, \"input_1\": train_tensor_e,\n                                                       \"input_2\": train_tensor_e, \"input_3\": train_tensor_e,\n                                                       \"input_4\": train_tensor_e, \"input_5\": train_tensor_e,\n                                                       \"input_6\": train_tensor_6}, labels_train))\n\nif not build_in_augmentation:\n    dataset_train_e2 = tf.data.Dataset.from_tensor_slices(({\"input_0\": train_tensor_0_aug, \"input_1\": train_tensor_1_aug,\n                                                            \"input_2\": train_tensor_2_aug, \"input_3\": train_tensor_3_aug,\n                                                            \"input_4\": train_tensor_4_aug, \"input_5\": train_tensor_5_aug,\n                                                            \"input_6\": train_tensor_6}, labels_train))\n    dataset_train_e = dataset_train_e.concatenate(dataset_train_e2)\n\ndataset_val_orig = tf.data.Dataset.from_tensor_slices(({\"input_0\": val_tensor_0, \"input_1\": val_tensor_1,\n                                                        \"input_2\": val_tensor_2, \"input_3\": val_tensor_3,\n                                                        \"input_4\": val_tensor_4, \"input_5\": val_tensor_5,\n                                                        \"input_6\": val_tensor_6}, labels_val))\n\ndataset_val_e = tf.data.Dataset.from_tensor_slices(({\"input_0\": val_tensor_0, \"input_1\": val_tensor_e,\n                                                     \"input_2\": val_tensor_e, \"input_3\": val_tensor_e,\n                                                     \"input_4\": val_tensor_e, \"input_5\": val_tensor_e,\n                                                     \"input_6\": val_tensor_6}, labels_val))\n\n# Create dataset for testing (dummy)\ndataset_test = tf.data.Dataset.from_tensor_slices(({\"input_0\": test_tensor_0, \"input_1\": test_tensor_1,\n                                                    \"input_2\": test_tensor_2, \"input_3\": test_tensor_3,\n                                                    \"input_4\": test_tensor_4, \"input_5\": test_tensor_5,\n                                                    \"input_6\": test_tensor_6}))\ndataset_test = dataset_test.batch(batch_size=1)\n\n# Concatenate original and augmented (wo ToF/THM data) datasets for train and validation\ndataset_train = dataset_train_orig.concatenate(dataset_train_e).shuffle(len(dataset_train_orig)*2,seed=SEED).batch(batch_size=batch_size, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\ndataset_val = dataset_val_orig.concatenate(dataset_val_e).batch(batch_size=1).prefetch(tf.data.AUTOTUNE)\ndataset_valbatch = dataset_val_orig.concatenate(dataset_val_e).batch(batch_size=batch_size_val, drop_remainder=True).prefetch(tf.data.AUTOTUNE)\n\nprint('Size of train dataset: '+ str(len(dataset_train_orig)*2))\nprint('Steps in train dataset per batch: '+ str(len(dataset_train)) + f' (in {round(len(dataset_train_orig)*2/len(dataset_train))} batches)')\nprint('Size of validation dataset: '+ str(len(dataset_val)))\nprint('Steps in validation dataset per batch: '+ str(len(dataset_valbatch)) + f' (in {round(len(dataset_val)/len(dataset_valbatch))} batches)')\nprint('Size of test dataset: '+ str(len(dataset_test)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:04:29.052814Z","iopub.execute_input":"2025-09-01T14:04:29.053023Z","iopub.status.idle":"2025-09-01T14:04:32.446535Z","shell.execute_reply.started":"2025-09-01T14:04:29.053006Z","shell.execute_reply":"2025-09-01T14:04:32.445570Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Build and compile neural network","metadata":{}},{"cell_type":"markdown","source":"The networks for the 2D (IMU) and 3D data (ToF/THM) was inspired by the SENet architecture (https://arxiv.org/abs/1709.01507). For the subject related data a conventional DNN network have been used.","metadata":{}},{"cell_type":"markdown","source":"![](https://tse1.mm.bing.net/th/id/OIP.D7VQxCrqzHxrPf8554Nm1wHaIQ?r=0&cb=thfvnext&rs=1&pid=ImgDetMain&o=7&rm=3)","metadata":{}},{"cell_type":"code","source":"## Define model branaches for 1D/2D/3D data\n\ndef SE_block(input_se, filters, name):\n    x = input_se\n    if input_se.ndim == 5:\n        x = tf.keras.layers.GlobalAveragePooling3D(name=name+'_se_gap3d')(x)\n    else:\n        x = tf.keras.layers.GlobalAveragePooling2D(name=name+'_se_gap2d')(x)\n    x = tf.keras.layers.Dense(units=int(filters/16), activation=\"relu\", kernel_initializer='he_normal', name=name+'_se_den')(x)\n    x = tf.keras.layers.Dense(units=filters, activation=\"sigmoid\", name=name+'_se_fin')(x)\n    if input_se.ndim == 5:\n        x = tf.keras.layers.Reshape((1, 1, 1, filters))(x)\n    else:\n        x = tf.keras.layers.Reshape((1, 1, filters))(x)\n    return x\n\n# Define 3D CNN model for ToF and THM sensor data\ndef Conv3D_model(input_conv3d, blocks_conv3d, filters_base_conv3d, name):\n    reg = None #L2(1e-4)\n    do = 0.15\n    x = input_conv3d\n    for i, filters in enumerate([filters_base_conv3d*(2**j) for j in range(blocks_conv3d)]):\n        if i < 3:\n            kernel = (3,3,6)\n            padding = 'same'\n            pool = (1,1,2)\n        elif i < 5:\n            kernel = (2,2,3)\n            padding = 'same'\n            pool = (2,2,2)\n        else:\n            kernel = (2,2,2)\n            padding = 'same'\n            pool = (2,2,2)\n        x_sc = x\n        x = tf.keras.layers.Conv3D(filters=filters, kernel_size=kernel, padding=padding,\n                                   activation=None, kernel_regularizer=reg, name=name+f'lay{i+1}1')(x)\n        x = tf.keras.layers.BatchNormalization(name=name+f'bn{i+1}1')(x)\n        x = tf.keras.layers.ReLU(name=name+f'relu{i+1}1')(x)\n        x = tf.keras.layers.Conv3D(filters=filters, kernel_size=kernel, padding=padding,\n                                   activation=None, kernel_regularizer=reg, name=name+f'lay{i+1}2')(x)\n        x = tf.keras.layers.BatchNormalization(name=name+f'bn{i+1}2')(x)\n        x_sc = tf.keras.layers.Conv3D(filters=filters, kernel_size=1, padding=padding,\n                                      activation=None, kernel_regularizer=reg, name=name+f'laysc{i+1}')(x_sc)\n        x_sc = tf.keras.layers.BatchNormalization(name=name+f'bnsc{i+1}')(x_sc)\n        x_se = SE_block(x, filters, name=name+f'layse{i+1}')\n        \n        x = tf.keras.layers.Multiply()([x, x_se])\n        x = tf.keras.layers.Add(name=name+f'add{i+1}')([x, x_sc])\n        x = tf.keras.layers.ReLU(name=name+f'relu{i+1}f')(x)\n        x = tf.keras.layers.MaxPool3D(pool_size=pool, name=name+f'mp{i+1}')(x)\n        x = tf.keras.layers.Dropout(do, name=name+f'do{i+1}')(x)\n    out = tf.keras.layers.GlobalAveragePooling3D(name=name+'gap')(x)\n    model = tf.keras.Model(inputs=input_conv3d, outputs=out, name=name)\n    return model\n\n# Define 2D CNN model for position and orientation sensor data\ndef Conv2D_model(input_conv2d, blocks_conv2d, filters_base_conv2d, name):\n    reg = None #L2(1e-4)\n    do = 0.15\n    x = input_conv2d\n    for i, filters in enumerate([filters_base_conv2d*(2**j) for j in range(blocks_conv2d)]):\n        if i < 3:\n            kernel = (3,6)\n            padding = 'same'\n            pool = (1,2)\n        elif i < 5:\n            kernel = (2,2)\n            padding = 'same'\n            pool = (2,2)\n        else:\n            kernel = (1,2)\n            padding = 'same'\n            pool = (1,2)\n        x_sc = x\n        x = tf.keras.layers.Conv2D(filters=filters, kernel_size=kernel, padding=padding,\n                                   activation=None, kernel_regularizer=reg, name=name+f'lay{i+1}1')(x)\n        x = tf.keras.layers.BatchNormalization(name=name+f'bn{i+1}1')(x)\n        x = tf.keras.layers.ReLU(name=name+f'relu{i+1}1')(x)\n        x = tf.keras.layers.Conv2D(filters=filters, kernel_size=kernel, padding=padding,\n                                   activation=None, kernel_regularizer=reg, name=name+f'lay{i+1}2')(x)\n        x = tf.keras.layers.BatchNormalization(name=name+f'bn{i+1}2')(x)\n        x_sc = tf.keras.layers.Conv2D(filters=filters, kernel_size=1, padding=padding,\n                                      activation=None, kernel_regularizer=reg, name=name+f'laysc{i+1}')(x_sc)\n        x_sc = tf.keras.layers.BatchNormalization(name=name+f'bnsc{i+1}')(x_sc)\n        x_se = SE_block(x, filters, name=name+f'layse{i+1}')\n        \n        x = tf.keras.layers.Multiply()([x, x_se])\n        x = tf.keras.layers.Add(name=name+f'add{i+1}')([x, x_sc])\n        x = tf.keras.layers.ReLU(name=name+f'relu{i+1}f')(x)\n        x = tf.keras.layers.MaxPool2D(pool_size=pool, name=name+f'mp{i+1}')(x)\n        #x = tf.keras.layers.Dropout(do, name=name+f'do{i+1}')(x)\n    out = tf.keras.layers.GlobalAveragePooling2D(name=name+'gap')(x)\n    model = tf.keras.Model(inputs=input_conv2d, outputs=out, name=name)\n    return model\n\n# Define 1D DNN model for subject related data\ndef DNN_model(input_dnn, layers_dnn, units_dnn, name):\n    reg = None #L2(1e-4)\n    do = 0.3\n    x = input_dnn\n    for i, units in enumerate([units_dnn for j in range(layers_dnn)]):\n        x = tf.keras.layers.Dense(units, activation=\"relu\", kernel_regularizer=reg,\n                                  kernel_initializer='he_normal', name=name+f'fc{i+1}')(x)\n        x = tf.keras.layers.BatchNormalization(name=name+f'bn{i+1}')(x)\n        x = tf.keras.layers.Dropout(do, name=name+f'do{i+1}')(x)\n    out = x\n    model = tf.keras.Model(inputs=input_dnn, outputs=out, name=name)\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:08:54.688772Z","iopub.execute_input":"2025-09-01T14:08:54.689054Z","iopub.status.idle":"2025-09-01T14:08:54.709181Z","shell.execute_reply.started":"2025-09-01T14:08:54.689036Z","shell.execute_reply":"2025-09-01T14:08:54.708514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Build 1D/2D/3D CNN network\n\ndef build_network(hp):\n    # define the sets of inputs\n    input_0 = tf.keras.Input(shape=(7, seq_len,1), name='input_0')\n    input_1 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_1')\n    input_2 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_2')\n    input_3 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_3')\n    input_4 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_4')\n    input_5 = tf.keras.Input(shape=(8, 8, seq_len, 2), name='input_5')\n    input_6 = tf.keras.Input(shape=(7,), name='input_6')\n\n    # data augmentation layer (shift time series data along time axis)\n    max_roll_layer = 0 # hp.Int(name='max_roll_layer', min_value=30, max_value=40, step=5, default=30)\n    max_acc_scale = 1 # hp.Float(name='max_acc_scale', min_value=1, max_value=4, step=0.5, sampling=\"linear\", default=3)\n    max_t_scale = 2 # hp.Float(name='max_t_scale', min_value=1, max_value=4, step=0.5, sampling=\"linear\", default=3)\n    if build_in_augmentation:\n        input_0a, input_1a, input_2a, input_3a, input_4a, input_5a = TranslationLayer(max_roll_layer, max_acc_scale, max_t_scale)(\n            [input_0, input_1, input_2, input_3, input_4, input_5])\n    else:\n        input_0a, input_1a, input_2a, input_3a, input_4a, input_5a = input_0, input_1, input_2, input_3, input_4, input_5\n        \n    # define the sets of branches for Conv3D tensors (ToF and THM sensor data)\n    blocks_conv3d = 6\n    filters_base_conv3d = 16\n    conv3d_model_1_out = Conv3D_model(input_1, blocks_conv3d, filters_base_conv3d, name='conv3d_model_1')(input_1a)\n    conv3d_model_2_out = Conv3D_model(input_2, blocks_conv3d, filters_base_conv3d, name='conv3d_model_2')(input_2a)\n    conv3d_model_3_out = Conv3D_model(input_3, blocks_conv3d, filters_base_conv3d, name='conv3d_model_3')(input_3a)\n    conv3d_model_4_out = Conv3D_model(input_4, blocks_conv3d, filters_base_conv3d, name='conv3d_model_4')(input_4a)\n    conv3d_model_5_out = Conv3D_model(input_5, blocks_conv3d, filters_base_conv3d, name='conv3d_model_5')(input_5a)\n    \n    # define branch for 2D tensor (IMU sensor data)\n    blocks_conv2d = 6\n    filters_base_conv2d = 32\n    data2d_model_out = Conv2D_model(input_0, blocks_conv2d, filters_base_conv2d, name='data2d_model')(input_0a)\n    \n    # define branch for 1D tensor (subject related data)\n    layers_dnn = 3\n    units_dnn = 256\n    data1d_model_out = DNN_model(input_6, layers_dnn, units_dnn, name='data1d_model')(input_6)\n    \n    # model_3ddummy = Conv3D_model(input_1, blocks_conv3d, filters_base_conv3d, name='conv3d_model_1')\n    # model_2ddummy = Conv2D_model(input_0, blocks_conv2d, filters_base_conv2d, name='data2d_model')\n    # model_3ddummy.summary()\n    # model_2ddummy.summary()\n    \n    # combine the output of the branches\n    z = tf.keras.layers.Concatenate(name='concat_all')([conv3d_model_1_out, conv3d_model_2_out, conv3d_model_3_out,\n                                                        conv3d_model_4_out, conv3d_model_5_out, data2d_model_out,\n                                                        data1d_model_out])\n    \n    # apply FC layer(s) and then a regression prediction on the combined outputs\n    reg = None #L2(1e-4)\n    do = 0.30\n    layers_final = 2\n    units_final = 512\n    for i, units in enumerate([units_final for j in range(layers_final)]):\n        z = tf.keras.layers.Dense(units, activation=\"relu\", kernel_regularizer=reg,\n                                  kernel_initializer='he_normal', name=f'final_fc{i+1}')(z)\n        z = tf.keras.layers.BatchNormalization(name=f'final_bn{i+1}')(z)\n        z = tf.keras.layers.Dropout(do, name=f'final_do{i+1}')(z)\n    out = tf.keras.layers.Dense(len(class2bfrb.keys()), activation=\"softmax\", kernel_regularizer=reg, name='prediction')(z)\n    model = tf.keras.Model(inputs=[input_0, input_1, input_2, input_3, input_4, input_5, input_6], outputs=out)\n\n    lr_tune = 5e-4 # hp.Float(name='learning_rate', min_value=5e-5, max_value=5e-3, sampling='log', default=5e-4)\n    optimizer = tf.keras.optimizers.Adam(lr_tune)\n    loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n    model.compile(optimizer=optimizer, loss=loss_fn, metrics=['accuracy'], run_eagerly=False)\n    return model\n\nwith strategy.scope():\n    model = build_network(kt.HyperParameters())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:26:10.950702Z","iopub.execute_input":"2025-09-01T14:26:10.951246Z","iopub.status.idle":"2025-09-01T14:26:12.422880Z","shell.execute_reply.started":"2025-09-01T14:26:10.951224Z","shell.execute_reply":"2025-09-01T14:26:12.421946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Explore model architecture\n\nmodel.summary()\ntf.keras.utils.plot_model(model, to_file='model_architecture.png', show_shapes=True, show_dtype=False,\n                          show_layer_names=True, show_layer_activations=True, show_trainable=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:26:13.596635Z","iopub.execute_input":"2025-09-01T14:26:13.596919Z","iopub.status.idle":"2025-09-01T14:26:15.444319Z","shell.execute_reply.started":"2025-09-01T14:26:13.596899Z","shell.execute_reply":"2025-09-01T14:26:15.443434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Training","metadata":{}},{"cell_type":"code","source":"## Training parameters\n\nepochs = 400\nTUNING = False\nTRAINING = False\nCUSTOM_TRAINING = False\nFINETUNING = False\n\n## Callback functions\nlr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(factor=0.1, patience=8, verbose=1, monitor='val_accuracy')\nearly_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=16, verbose=1, monitor='val_accuracy', restore_best_weights=True)\nlr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-5 * 10**(epoch / 20)) # Find starting learning","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:21:10.736321Z","iopub.execute_input":"2025-09-01T14:21:10.736882Z","iopub.status.idle":"2025-09-01T14:21:10.741480Z","shell.execute_reply.started":"2025-09-01T14:21:10.736858Z","shell.execute_reply":"2025-09-01T14:21:10.740754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Load pre-trained weights\n\nif not TRAINING and not TUNING and not CUSTOM_TRAINING:\n    model.load_weights('/kaggle/input/cmi-59x/CMI_5_10_2.weights.h5')\n    print('Model weights have been loaded!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:05:07.570975Z","iopub.execute_input":"2025-09-01T14:05:07.571245Z","iopub.status.idle":"2025-09-01T14:05:07.575282Z","shell.execute_reply.started":"2025-09-01T14:05:07.571225Z","shell.execute_reply":"2025-09-01T14:05:07.574421Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train model\n\nif TRAINING or FINETUNING:\n    history = model.fit(dataset_train, validation_data=dataset_valbatch, epochs=epochs, callbacks=[lr_scheduler, early_stopping_cb])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:05:08.900798Z","iopub.execute_input":"2025-09-01T14:05:08.901062Z","iopub.status.idle":"2025-09-01T14:05:08.905036Z","shell.execute_reply.started":"2025-09-01T14:05:08.901041Z","shell.execute_reply":"2025-09-01T14:05:08.904424Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Tuner\n\ni_TunerTyp = 1 # Choose desired tuner type: {1: 'grid', 2: 'random', 3: 'hyper'}\nTunerStr = {1: 'grid', 2: 'random', 3: 'hyper'}\n\ntuner_grid = kt.GridSearch(hypermodel=build_network, objective='val_accuracy',\n                           max_trials=10, max_consecutive_failed_trials=1,\n                           overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\ntuner_random = kt.RandomSearch(hypermodel=build_network, objective='val_accuracy',\n                               max_trials=10, executions_per_trial=1,\n                               overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\ntuner_hyper = kt.Hyperband(hypermodel=build_network, objective='val_accuracy',\n                           max_epochs=90, factor=5, hyperband_iterations=1,\n                           overwrite=True, directory=\"tuner\", project_name=\"CMI\", distribution_strategy = strategy)\n\ntuner = globals()[f'tuner_{TunerStr[i_TunerTyp]}']\ntuner.search_space_summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:05:11.210887Z","iopub.execute_input":"2025-09-01T14:05:11.211624Z","iopub.status.idle":"2025-09-01T14:05:14.489184Z","shell.execute_reply.started":"2025-09-01T14:05:11.211598Z","shell.execute_reply":"2025-09-01T14:05:14.488372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Train model by Tuner\n\nif TUNING:\n    tuner.search(dataset_train, validation_data=dataset_valbatch, epochs=epochs, callbacks=[lr_scheduler, early_stopping_cb])\n\n    best_models = tuner.get_best_models(num_models=2)\n    model = best_models[0]\n    model.summary()\n    tuner.results_summary()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:05:14.490360Z","iopub.execute_input":"2025-09-01T14:05:14.490628Z","iopub.status.idle":"2025-09-01T14:05:14.494988Z","shell.execute_reply.started":"2025-09-01T14:05:14.490610Z","shell.execute_reply":"2025-09-01T14:05:14.494374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Custom training loop\n\nif CUSTOM_TRAINING:\n    train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n    val_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()\n    #loss_fn = tf.keras.losses.sparse_categorical_crossentropy\n    #optimizer = tf.keras.optimizers.Adam(0.0005)\n    loss_fn = model.loss\n    optimizer = model.optimizer\n    history = tf.keras.callbacks.History()\n    history.set_model(model)\n\n@tf.function\ndef train_step(x, y):\n    with tf.GradientTape() as tape:\n        logits = model(x, training=True)\n        loss_value = loss_fn(y, logits)\n    grads = tape.gradient(loss_value, model.trainable_weights)\n    optimizer.apply_gradients(zip(grads, model.trainable_weights))\n    train_acc_metric.update_state(y, logits)\n    return loss_value\n\n@tf.function\ndef test_step(x, y):\n    val_logits = model(x, training=False)\n    val_loss_value = loss_fn(y, val_logits)\n    val_acc_metric.update_state(y, val_logits)\n    return val_loss_value\n\ndef custom_train():\n    epochs = 200\n    patience_early = 11\n    patience_lr = 8\n    wait_early = 0\n    wait_lr = 0\n    best_acc_early = float(0.0)\n    best_acc_lr = float(0.0)\n    \n    for epoch in range(epochs): # len(dataset_train)\n        progBar = tf.keras.utils.Progbar(229, stateful_metrics=['train_loss', 'train_acc', 'val_loss', 'val_acc'])\n        history.on_train_begin(logs={'loss': 10.0, 'accuracy': 0.0,\n                                     'val_loss': 10.0, 'val_accuracy': 0.0,\n                                     'learning_rate': optimizer.learning_rate.numpy()})\n        \n        # Iterate over the batches of the dataset\n        loss_value_accum = 0\n        for step, (x_batch_train, y_batch_train) in enumerate(dataset_train):\n            loss_value_accum += train_step(x_batch_train, y_batch_train)\n            loss_value = loss_value_accum/(step+1)\n            train_acc = train_acc_metric.result()\n            # Update progress bar with training loss and accuracy\n            progBar.update(step, values=[('train_loss', loss_value.numpy().mean()), ('train_acc', train_acc.numpy())])\n    \n        # Run a validation loop at the end of each epoch.\n        for x_batch_val, y_batch_val in dataset_valbatch:\n            val_loss_value = test_step(x_batch_val, y_batch_val)\n        val_acc = val_acc_metric.result()\n        \n        # Update progress bar with validation loss and accuracy\n        progBar.update(step+1, values=[('train_loss', loss_value.numpy().mean()), ('train_acc', train_acc.numpy()),\n                                       ('val_loss', val_loss_value.numpy().mean()), ('val_acc', val_acc.numpy()),\n                                       ('learning_rate', optimizer.learning_rate.numpy()), ('epoch', int(epoch+1)),\n                                       ('wait', wait_early)], finalize=True)\n        history.on_epoch_end(int(epoch+1), logs={'loss': loss_value.numpy().mean(), 'accuracy': train_acc.numpy(),\n                                                 'val_loss': val_loss_value.numpy().mean(), 'val_accuracy': val_acc.numpy(),\n                                                 'learning_rate': optimizer.learning_rate.numpy()})\n\n        # Reset metrics at the end of each epoch\n        train_acc_metric.reset_state()\n        val_acc_metric.reset_state()\n        \n        # The early stopping strategy: stop the training if `val_acc` does not decrease over a certain number of epochs.\n        wait_early += 1\n        if val_acc > best_acc_early:\n            best_acc_early = val_acc\n            model.save_weights('cp_best.weights.h5')\n            best_epoch = epoch+1\n            wait_early = 0\n        if wait_early >= patience_early:\n            model.load_weights('cp_best.weights.h5')\n            print(f'Early stopping. Best model is restored from epoch {best_epoch}')\n            break\n        \n        wait_lr += 1\n        if val_acc > best_acc_lr:\n            best_acc_lr = val_acc\n            wait_lr = 0\n        if wait_lr >= patience_lr:\n            model.load_weights('cp_best.weights.h5')\n            optimizer.learning_rate = optimizer.learning_rate * 0.1\n            wait_lr = 0\n\nif CUSTOM_TRAINING:\n    custom_train()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-01T14:21:20.417778Z","iopub.execute_input":"2025-09-01T14:21:20.418047Z","iopub.status.idle":"2025-09-01T14:25:02.967922Z","shell.execute_reply.started":"2025-09-01T14:21:20.418027Z","shell.execute_reply":"2025-09-01T14:25:02.966842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Save weights of model\n\nif TRAINING or TUNING or FINETUNING or CUSTOM_TRAINING:\n    model.save_weights('CMI_5_10_4.weights.h5')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:36:37.029396Z","iopub.execute_input":"2025-07-29T08:36:37.029786Z","iopub.status.idle":"2025-07-29T08:36:40.494602Z","shell.execute_reply.started":"2025-07-29T08:36:37.029759Z","shell.execute_reply":"2025-07-29T08:36:40.489578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Evaluation","metadata":{}},{"cell_type":"code","source":"## Plot learning curves\n\nif TRAINING or FINETUNING or CUSTOM_TRAINING:\n    history_fil = {key: history.history[key] for key in ['accuracy', 'val_accuracy']}\n    history_fil2 = {key: history.history[key] for key in ['loss', 'val_loss']}\n    history_fil3 = {key: history.history[key] for key in ['learning_rate']}\n    \n    pd.DataFrame(history_fil).plot()\n    plt.ylabel(\"Accuracy\")\n    plt.xlabel(\"epochs\")\n    pd.DataFrame(history_fil2).plot()\n    plt.ylabel(\"Loss\")\n    plt.xlabel(\"epochs\")\n    plt.axis([10, len(history_fil2['val_loss']), 0, history_fil2['val_loss'][10]+0.1*history_fil2['val_loss'][10]])\n    pd.DataFrame(history_fil3).plot()\n    plt.ylabel(\"Learning rate\")\n    plt.xlabel(\"epochs\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-29T08:29:20.341901Z","iopub.execute_input":"2025-07-29T08:29:20.342253Z","iopub.status.idle":"2025-07-29T08:29:21.050343Z","shell.execute_reply.started":"2025-07-29T08:29:20.342206Z","shell.execute_reply":"2025-07-29T08:29:21.044356Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Calculate competition specific score\n\nfrom cmi_2025_metric_copy_for_import import CompetitionMetric\n\n# Get predicted labels for the validation set\nimages, labels = tuple(zip(*dataset_val))\n\nprint(model.evaluate(dataset_val))\nprobabilities = model.predict(dataset_val)\ny_val_pred = np.argmax(probabilities, axis=-1)\ny_val_true = np.squeeze(np.array(labels))\n\n# Map integer labels back to gesture strings\nval_pred_labels = pd.Series(y_val_pred).map(lambda i: id2class[i])\nval_true_labels = pd.Series(y_val_true).map(lambda i: id2class[i])\n\n# Build DataFrames for the metric\nval_submission = pd.DataFrame({'gesture': val_pred_labels})\nval_solution = pd.DataFrame({'gesture': val_true_labels})\n\n# Run competition metric\nmetric = CompetitionMetric()\nscore = metric.calculate_hierarchical_f1(val_solution, val_submission)\nprint(f\"Estimated leaderboard (val) score: {score:.4f}\")\n\nprint(val_pred_labels)\nprint(val_true_labels)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Submission","metadata":{}},{"cell_type":"code","source":"## Init server\n\nimport kaggle_evaluation.cmi_inference_server\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    test = sequence.to_pandas()\n    if len(test.index) > seq_len:\n        print(len(test.index))\n        test = test[-seq_len:]\n    #print(len(test.index))\n\n    test_demographics = demographics.to_pandas()\n    test_tensor_0, test_tensor_1, test_tensor_2, test_tensor_3, test_tensor_4, test_tensor_5 = extract_time_series_data(test, False)\n    test_tensor_6, _ = extract_subject_related_data(test, test_demographics, scalers_sub, False)\n    dataset_test = tf.data.Dataset.from_tensor_slices(({\"input_0\": test_tensor_0,\n                                                        \"input_1\": test_tensor_1, \"input_2\": test_tensor_2,\n                                                        \"input_3\": test_tensor_3, \"input_4\": test_tensor_4,\n                                                        \"input_5\": test_tensor_5, \"input_6\": test_tensor_6}))\n    dataset_test = dataset_test.batch(batch_size=1)\n    \n    probabilities = model.predict(dataset_test, verbose=0)\n    predictions = np.argmax(probabilities, axis=-1)\n    prediction = np.argmax(probabilities, axis=-1)[0]\n    return id2class[prediction]\n\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"## Run server\n\nif not TRAINING and not TUNING and not CUSTOM_TRAINING:\n    if os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n        inference_server.serve()\n    else:\n        inference_server.run_local_gateway(\n            data_paths=(\n                '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n                '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n            )\n        )","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Experimenting code","metadata":{}},{"cell_type":"code","source":"# ## Explore max/min/average length of sequences\n# data_check = train[train['phase']=='Gesture'] # Max: 71\n# data_check = train # Max: 700, Min: 29, Avg: 71 ('SEQ_015261')\n# min_length = 1000\n# max_length = 0\n# for s, sequence_id in enumerate(data_check['sequence_id'].unique()):\n#     if s%1000 == 0: print(s) # Show progress\n#     data_check_fil = data_check[data_check['sequence_id'] == sequence_id]\n#     seq_length = len(data_check_fil.index)\n#     if seq_length < min_length:\n#         min_length = seq_length\n#         min_seq_id = sequence_id\n#     if seq_length > max_length:\n#         max_length = seq_length\n#         max_seq_id = sequence_id","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ## Plot learning curves for definition of start leraning rate\n# lrs = 1e-5 * (10 ** (np.arange(100) / 20)) # Define the learning rate array\n# plt.figure(figsize=(10, 6)) # Set the figure size\n# plt.grid(True) # Set the grid\n# plt.semilogx(lrs, history.history[\"loss\"]) # Plot the loss in log scale\n# plt.tick_params('both', length=10, width=1, which='both') # Increase the tickmarks size\n# plt.axis([1e-5, 1e-0, 0, 10]) # Set the plot boundaries","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Draft backup code (for debuging purposes)","metadata":{}},{"cell_type":"code","source":"# start = time.time()\n# TranslationLayer(20, 3.25, 1.5)([train_tensor_0 , train_tensor_1, train_tensor_2, train_tensor_3, train_tensor_4, train_tensor_5],\n#                               training=True)\n# end = time.time()\n# print(end - start)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}