{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"b1930f2f","cell_type":"markdown","source":"# BFRB Detection EDA: Deep Dive into Wrist Sensor Data (CMI Competition)\n\nIn this notebook, we embark on a comprehensive exploration of the Helios wrist sensor dataset from the CMI body-focused repetitive behavior (BFRB) detection Kaggle competition. This notebook provides tools for investigating thermal, proximity, and motion signals to discover patterns distinguishing subjects' gestures to lay the foundation for engineering features for classification machine learning models.\n\n## Table of Contents\n- [1. Data Overview](#1-data-overview)\n- [2. Sequence Analysis](#2-sequence-analysis)\n- [3. Phase Sequences](#3-phase-sequences)\n- [4. Missing Data](#4-missing-data)\n- [5. Demographics Data](#5-demographics-data)\n- [6. Time Series Data](#6-time-series-data)\n  - [6.1 IMU Data](#6.1-imu-data)\n  - [6.2 Thermopile and Time-of-Flight Sensor Data](#6.2-thermopile-and-time-of-flight-sensor-data)","metadata":{}},{"id":"e99ffc06","cell_type":"code","source":"import os\nfrom collections import Counter\nfrom enum import Enum\nfrom pathlib import Path\nfrom typing import Any, cast\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib.backends.backend_pdf import PdfPages\nfrom matplotlib.figure import Figure\nfrom pydantic import BaseModel, ConfigDict\nfrom scipy.fft import rfft, rfftfreq\nfrom scipy.spatial.transform import Rotation as R\n\n# Set to true if you want to export plots to PDF files.\nEXPORT_PLOTS_TO_PDF = False\n\nDATA_DIR = Path(\"../kaggle/input/cmi-detect-behavior-with-sensor-data\").resolve()\nPLOT_DIR = Path(\"../plots\").resolve()\n\nif os.environ.get(\"KAGGLE_KERNEL_RUN_TYPE\"):\n    import warnings\n\n    warnings.simplefilter(action=\"ignore\", category=FutureWarning)\n    DATA_DIR = Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\").resolve()\n    PLOT_DIR = Path(\"/kaggle/working/plots\").resolve()\n\nPLOT_DIR.mkdir(exist_ok=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:07.838088Z","iopub.execute_input":"2025-08-28T07:14:07.838438Z","iopub.status.idle":"2025-08-28T07:14:07.848183Z","shell.execute_reply.started":"2025-08-28T07:14:07.838415Z","shell.execute_reply":"2025-08-28T07:14:07.847255Z"}},"outputs":[],"execution_count":null},{"id":"12cce640","cell_type":"markdown","source":"## 1. Data Overview\n\nLet's begin by importing the data and getting a basic idea of the shape, data types, and features of the training table.","metadata":{}},{"id":"5f693aae","cell_type":"code","source":"train_data = pd.read_csv(DATA_DIR.joinpath(\"train.csv\"))\ntrain_demographics_data = pd.read_csv(DATA_DIR.joinpath(\"train_demographics.csv\"))\ntest_data = pd.read_csv(DATA_DIR.joinpath(\"test.csv\"))\ntest_demographics_data = pd.read_csv(DATA_DIR.joinpath(\"test_demographics.csv\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:07.849606Z","iopub.execute_input":"2025-08-28T07:14:07.849959Z","iopub.status.idle":"2025-08-28T07:14:29.952859Z","shell.execute_reply.started":"2025-08-28T07:14:07.849928Z","shell.execute_reply":"2025-08-28T07:14:29.952051Z"}},"outputs":[],"execution_count":null},{"id":"a4a4e0a4","cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:29.954266Z","iopub.execute_input":"2025-08-28T07:14:29.954578Z","iopub.status.idle":"2025-08-28T07:14:29.974943Z","shell.execute_reply.started":"2025-08-28T07:14:29.954551Z","shell.execute_reply":"2025-08-28T07:14:29.974059Z"}},"outputs":[],"execution_count":null},{"id":"f2e48232","cell_type":"markdown","source":"The above table has 341 columns, causing many to be hidden. To get a better understanding of the scale of data within each feature group, let's filter out the columns that are just components of the data in other columns (e.g. `acc_y` is the y-component of the IMU sensor's acceleration measurements).","metadata":{}},{"id":"cb89b646","cell_type":"code","source":"def get_filtered_columns(df: pd.DataFrame, redundant_prefixes: set[str]) -> list[str]:\n    \"\"\"Filters out duplicate column groups based on redundant prefixes.\n\n    Args:\n        df (pd.DataFrame): Training data frame.\n        redundant_prefixes (set[str]): A set of column name prefixes that are considered redundant if they appear more\n            than once. Only the first column with each redundant prefix will be kept.\n\n    Returns:\n        list[str]: A filtered list of column names, preserving order, with redundant prefix groups reduced to a single\n            representative column.\n    \"\"\"\n    seen_prefixes: set[str] = set()\n    filtered_columns: list[str] = []\n    for col in df.columns:\n        prefix = col.split(\"_\")[0]\n        if prefix in redundant_prefixes:\n            if prefix in seen_prefixes:\n                continue\n            seen_prefixes.add(prefix)\n        filtered_columns.append(col)\n    return filtered_columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:29.976031Z","iopub.execute_input":"2025-08-28T07:14:29.976465Z","iopub.status.idle":"2025-08-28T07:14:29.992421Z","shell.execute_reply.started":"2025-08-28T07:14:29.976436Z","shell.execute_reply":"2025-08-28T07:14:29.991614Z"}},"outputs":[],"execution_count":null},{"id":"f3d515ca","cell_type":"code","source":"redundant_prefixes = {\"acc\", \"rot\", \"thm\", \"tof\"}\nfiltered_columns = get_filtered_columns(train_data, redundant_prefixes)\ntrain_data[filtered_columns].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:29.994480Z","iopub.execute_input":"2025-08-28T07:14:29.994755Z","iopub.status.idle":"2025-08-28T07:14:30.064884Z","shell.execute_reply.started":"2025-08-28T07:14:29.994736Z","shell.execute_reply":"2025-08-28T07:14:30.064090Z"}},"outputs":[],"execution_count":null},{"id":"60d4f084","cell_type":"markdown","source":"## 2. Sequence Analysis\n\nAccording to the [demo submission](https://www.kaggle.com/code/sohier/cmi-2025-demo-submission), the CMI inference server provides one sequence from the test set at a time to our model for prediction. Let us start our investigation by observing how many sequences there are and getting a sense for how many rows there are per sequence. The tail of the table will show us the index of the latest sequence if it is sorted.","metadata":{}},{"id":"16b24ea6","cell_type":"code","source":"train_data.tail()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:30.065628Z","iopub.execute_input":"2025-08-28T07:14:30.065891Z","iopub.status.idle":"2025-08-28T07:14:30.087373Z","shell.execute_reply.started":"2025-08-28T07:14:30.065872Z","shell.execute_reply":"2025-08-28T07:14:30.086349Z"}},"outputs":[],"execution_count":null},{"id":"826d96db","cell_type":"markdown","source":"We will check if the table is correctly sorted, if the sequences are sequential, and what the first and last sequence indices are.","metadata":{}},{"id":"c8e609f1","cell_type":"code","source":"def get_seq_no(df: pd.DataFrame, index: int) -> int:\n    \"\"\"Returns the sequence number from an index of the training data frame.\n\n    Args:\n        df (pd.DataFrame): Training data frame.\n        index (int): Data frame row index.\n\n    Returns:\n        int: Sequence number.\n    \"\"\"\n    return int(str(df.iloc[index][\"sequence_id\"]).removeprefix(\"SEQ_\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:30.088383Z","iopub.execute_input":"2025-08-28T07:14:30.088746Z","iopub.status.idle":"2025-08-28T07:14:30.105057Z","shell.execute_reply.started":"2025-08-28T07:14:30.088680Z","shell.execute_reply":"2025-08-28T07:14:30.103909Z"}},"outputs":[],"execution_count":null},{"id":"bc2dd48e","cell_type":"code","source":"if train_data.equals(train_data.sort_values([\"sequence_id\", \"sequence_counter\"])):\n    print(f\"train_data is sorted from sequence {get_seq_no(train_data, 0)} to {get_seq_no(train_data, -1)}\")\n\nunique_ids = train_data[\"sequence_id\"].drop_duplicates().str.extract(r\"SEQ_(\\d+)\")[0].astype(int).tolist()\n\nif unique_ids != list(range(get_seq_no(train_data, 0), get_seq_no(train_data, -1) + len(unique_ids))):\n    print(\"train_data's sequences are not sequential.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:30.106045Z","iopub.execute_input":"2025-08-28T07:14:30.106651Z","iopub.status.idle":"2025-08-28T07:14:32.289436Z","shell.execute_reply.started":"2025-08-28T07:14:30.106626Z","shell.execute_reply":"2025-08-28T07:14:32.288514Z"}},"outputs":[],"execution_count":null},{"id":"a74371c1","cell_type":"markdown","source":"We can see that the sequences are not sequential (e.g. 7, 8, 9, ...), but they are sorted from 7 to 65,531. Since the sequences are not sequential, let's see if there are any large gaps in the sequence numbers. To do this we will turn all the sequence IDs into integers and plot them in  a histogram.","metadata":{}},{"id":"5b402f58","cell_type":"code","source":"plt.figure(figsize=(10, 5))\n\nbin_length = 1_000\nbins = list(range(0, 65_000, bin_length))\ncounts, _bin_edges, _patches = plt.hist(unique_ids, bins=bins, edgecolor=\"black\")\ncounts = pd.Series(counts)\n\nprint(\n    f\"Each bin of length {bin_length} contains an average of {int(counts.mean())} \"\n    f\"({int(counts.mean() / bin_length * 100)}%) sequences with a standard deviation of {int(counts.std())} and \"\n    f\"coefficient of variation of {int(counts.std() / counts.mean() * 100)}%.\"\n)\n\nplt.title(\"Distribution of Sequence Numbers\")\nplt.xlabel(\"Sequence Number\")\nplt.ylabel(\"Count\")\nplt.grid(axis=\"y\", linestyle=\":\", alpha=0.5)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:32.290455Z","iopub.execute_input":"2025-08-28T07:14:32.290822Z","iopub.status.idle":"2025-08-28T07:14:32.620347Z","shell.execute_reply.started":"2025-08-28T07:14:32.290779Z","shell.execute_reply":"2025-08-28T07:14:32.619414Z"}},"outputs":[],"execution_count":null},{"id":"1f54457d","cell_type":"markdown","source":"Our coefficient of variation of under 10% means that although our sequence IDs are not sequential (we are missing 88% of them) they are still fairly evenly distributed. Next let's observe the uniformity of our sequence lengths.","metadata":{}},{"id":"4cff2214","cell_type":"code","source":"sequence_lens = train_data.groupby(\"sequence_id\")[\"sequence_counter\"].max()\n\nprint(f\"Total number of sequences: {len(sequence_lens)}\")\nprint(f\"Shortest sequence: {min(sequence_lens)}\")\nprint(f\"Longest sequence: {max(sequence_lens)}\")\n\nplt.figure(figsize=(10, 5))\n\nbins = list(range(0, 701, 10))\ncounts, bin_edges, _patches = plt.hist(sequence_lens, bins=bins, edgecolor=\"black\")\nmax_count = np.max(counts)\nmax_bin_index = np.argmax(counts)\nmax_bin_range = (int(bin_edges[max_bin_index]), int(bin_edges[max_bin_index + 1]))\n\nprint(\n    f\"Most common sequence length range {max_bin_range} occurred {int(max_count)} times or \"\n    f\"{int(max_count / sum(counts) * 100)}% of the time.\"\n)\n\nplt.xlabel(\"Length of sequence\")\nplt.ylabel(\"Frequency\")\nplt.title(\"Histogram of sequence lengths\")\nplt.grid(axis=\"y\", linestyle=\":\", alpha=0.5)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:32.621245Z","iopub.execute_input":"2025-08-28T07:14:32.621564Z","iopub.status.idle":"2025-08-28T07:14:32.942895Z","shell.execute_reply.started":"2025-08-28T07:14:32.621535Z","shell.execute_reply":"2025-08-28T07:14:32.941990Z"}},"outputs":[],"execution_count":null},{"id":"6f0a81ab","cell_type":"markdown","source":"The most common sequence length is in the 50-60 element range, appearing about one-third of the time, but the sequences in general vary from 28 rows to 699 per sequence. \n\nNow that we know the sequences are ordered and have a better sense of their length distribution, let's see which features in the training dataset are not in the test dataset.","metadata":{}},{"id":"f6229f98","cell_type":"code","source":"columns_only_in_test = [col for col in test_data.columns if col not in train_data.columns]\ncolumns_only_in_train = [col for col in train_data.columns if col not in test_data.columns]\nif not columns_only_in_test:\n    print(\"All columns in the test data exist in the training data.\")\nprint(f\"The following columns are ONLY in the training data and not in the test data: {columns_only_in_train}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:32.945626Z","iopub.execute_input":"2025-08-28T07:14:32.945902Z","iopub.status.idle":"2025-08-28T07:14:32.952148Z","shell.execute_reply.started":"2025-08-28T07:14:32.945881Z","shell.execute_reply":"2025-08-28T07:14:32.951330Z"}},"outputs":[],"execution_count":null},{"id":"f5a2b875","cell_type":"markdown","source":"We can see that the features `sequence_type` (e.g. `Target`), `orientation` (e.g. `Seated Lean Non Dom - FACE DOWN`), `behavior` (e.g. `Relaxes and moves hand to target location`), `phase` (e.g. `Transition`), and `gesture` (e.g. `Cheek - pinch skin`) are _only_ in the training data and not in the test data. This makes sense since CMI is trying to predict gestures based solely on sensor data and these columns are specific to their research trials. The column we are tasked to predict is `gesture`, however some care may need to be taken with regard to these other features to optimize the score. \n\nThe [overview page](www.kaggle.com/competitions/cmi-detect-behavior-with-sensor-data/overview/evaluation) states that the final score is the average of the binary F1 (sequence type) and the macro F1 (gesture) scores. This makes me wonder if it might be advantageous for our loss function to penalize predictions of sequence type with comparable weight to the gesture.\n\nWe should also investigate how orientation affects sensor data, taking great care to make sure the model recognizes differing sensor data based on orientation as the same gestures. Another consideration is that may want to have the model distinguish between the transition and gesture phases since transitions may be similar between trials while we are  only trying to distinguish between gestures.\n\n## 3. Phase Sequences\n\nLet's start by observing all the unique values in the phase column.","metadata":{}},{"id":"7325ddc7","cell_type":"code","source":"train_data[\"phase\"].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:32.952925Z","iopub.execute_input":"2025-08-28T07:14:32.953121Z","iopub.status.idle":"2025-08-28T07:14:32.996635Z","shell.execute_reply.started":"2025-08-28T07:14:32.953105Z","shell.execute_reply":"2025-08-28T07:14:32.995679Z"}},"outputs":[],"execution_count":null},{"id":"34e68b8a","cell_type":"markdown","source":"The competition overview claims participants were asked to start in a rest position (`Transition`), then pause and do nothing, then enact the gesture (`Gesture`). Let's first verify this sequence does in fact hold for all the data and see if any sequences might be missing either of these phases.","metadata":{}},{"id":"5f9c3b1a","cell_type":"code","source":"PHASE_ORDER = {\"Transition\": 0, \"Gesture\": 1}\n\n\ndef does_phase_dec(seq: pd.DataFrame) -> bool:\n    \"\"\"Returns whether the phases for a given sequence decrease (i.e `Gesture` -> `Transition`).\n\n    Args:\n        seq (pd.DataFrame): Training data sequence.\n\n    Returns:\n        bool: Whether the phases for a given sequence decrease.\n    \"\"\"\n    mapped: pd.Series = seq[\"phase\"].map(PHASE_ORDER)\n    return not mapped.is_monotonic_increasing","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:32.997611Z","iopub.execute_input":"2025-08-28T07:14:32.997926Z","iopub.status.idle":"2025-08-28T07:14:33.003530Z","shell.execute_reply.started":"2025-08-28T07:14:32.997899Z","shell.execute_reply":"2025-08-28T07:14:33.002608Z"}},"outputs":[],"execution_count":null},{"id":"74dc829f","cell_type":"code","source":"invalid_sequences = train_data.groupby(\"sequence_id\").filter(does_phase_dec)\n\nif invalid_sequences.empty:\n    print(\"All sequences follow the correct Transition → Gesture order.\")\n\nmissing_phase_info = train_data.groupby(\"sequence_id\")[\"phase\"].apply(lambda phases: set(PHASE_ORDER) - set(phases))\n\nsequences_missing_phases = missing_phase_info[missing_phase_info.apply(bool)]\n\nfor seq_id, missing in sequences_missing_phases.items():\n    print(f\"Sequence {seq_id} is missing: {', '.join(missing)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:33.004499Z","iopub.execute_input":"2025-08-28T07:14:33.005411Z","iopub.status.idle":"2025-08-28T07:14:37.456283Z","shell.execute_reply.started":"2025-08-28T07:14:33.005390Z","shell.execute_reply":"2025-08-28T07:14:37.455279Z"}},"outputs":[],"execution_count":null},{"id":"1586da87","cell_type":"markdown","source":"We can observe that all features do follow the correct phase order, however one of the sequences (11,975) is completely missing a gesture! It may therefore be worth completely removing this sequence from the training data.\n\nNow let's explore the behavior column.","metadata":{}},{"id":"b30a69db","cell_type":"code","source":"train_data[\"behavior\"].unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:37.457553Z","iopub.execute_input":"2025-08-28T07:14:37.458476Z","iopub.status.idle":"2025-08-28T07:14:37.509452Z","shell.execute_reply.started":"2025-08-28T07:14:37.458454Z","shell.execute_reply":"2025-08-28T07:14:37.508560Z"}},"outputs":[],"execution_count":null},{"id":"9497b30a","cell_type":"markdown","source":"Interestingly enough, there appear to be two different behaviors that might be the same in meaning, just with different labels (`Relaxes and moves hand to target location` and `Moves hand to target location`). Let's explore a little bit more about what unique orders of the behaviors we see as well as which phases they line up with.","metadata":{}},{"id":"6b2319ba","cell_type":"code","source":"def behavior_phase_seq(seq: pd.DataFrame) -> Any:\n    \"\"\"Returns all ordered pairs of behavior and phase for a given sequence from the training data.\n\n    Args:\n        seq (pd.DataFrame): A sequence from the training data.\n\n    Returns:\n        Any: Ordered pairs of behavior and phase for a given sequence.\n    \"\"\"\n    unique_pairs = seq[[\"behavior\", \"phase\"]].drop_duplicates()\n    return tuple(map(tuple, unique_pairs.to_numpy()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:37.510753Z","iopub.execute_input":"2025-08-28T07:14:37.511104Z","iopub.status.idle":"2025-08-28T07:14:37.515797Z","shell.execute_reply.started":"2025-08-28T07:14:37.511082Z","shell.execute_reply":"2025-08-28T07:14:37.514896Z"}},"outputs":[],"execution_count":null},{"id":"47434842","cell_type":"code","source":"behavior_phase_orders = (\n    train_data[[\"sequence_id\", \"behavior\", \"phase\"]]\n    .groupby(\"sequence_id\")\n    .apply(behavior_phase_seq, include_groups=False)\n)\n\norder_counts = Counter(behavior_phase_orders)\n\nfor i, (order, count) in enumerate(order_counts.items(), 1):\n    print(f\"{i}. {order} → {count} sequences\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:37.516935Z","iopub.execute_input":"2025-08-28T07:14:37.517213Z","iopub.status.idle":"2025-08-28T07:14:44.857824Z","shell.execute_reply.started":"2025-08-28T07:14:37.517184Z","shell.execute_reply":"2025-08-28T07:14:44.856898Z"}},"outputs":[],"execution_count":null},{"id":"f7f3dd97","cell_type":"markdown","source":"So of our 8,151 sequences, approximately half of them contain possibly the same sequences as the other half by meaning, but different labels. As we observed before, we have one sequence without any gesture phase at all. If these two behaviors really are the same, and if we want to train our model to predict this feature, it may be worth replacing all of one type with the other.\n\n## 4. Missing Data\n\nGoing back to each of these filtered column types, let's investigate which ones have missing data and how many rows they might be missing.","metadata":{}},{"id":"c2060727","cell_type":"code","source":"train_data[filtered_columns].isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:44.859096Z","iopub.execute_input":"2025-08-28T07:14:44.859361Z","iopub.status.idle":"2025-08-28T07:14:45.171383Z","shell.execute_reply.started":"2025-08-28T07:14:44.859342Z","shell.execute_reply":"2025-08-28T07:14:45.170523Z"}},"outputs":[],"execution_count":null},{"id":"7bc55e6d","cell_type":"markdown","source":"It appears the IMU orientation data, thermopile temperature sensor data, and time-of-flight distance sensor data are all missing rows. Let's verify that these are the only three types missing data out of all columns in the original dataset.","metadata":{}},{"id":"387402f7","cell_type":"code","source":"def get_col_groups_missing_data(df: pd.DataFrame) -> set[str]:\n    \"\"\"Returns all column groups that contain missing data (e.g. thermopiles or time-of-flight sensor columns).\n\n    Args:\n        df (pd.DataFrame): Training data frame.\n\n    Returns:\n        set[str]: Collection of column groups with missing data.\n    \"\"\"\n    s = df.isnull().sum()\n    nullish_indices = s[s > 0].index.tolist()\n    index_prefixes: set[str] = set()\n\n    for i in nullish_indices:\n        index_prefixes.add(str(i).split(\"_\")[0])\n\n    return index_prefixes","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:45.172406Z","iopub.execute_input":"2025-08-28T07:14:45.172749Z","iopub.status.idle":"2025-08-28T07:14:45.178141Z","shell.execute_reply.started":"2025-08-28T07:14:45.172729Z","shell.execute_reply":"2025-08-28T07:14:45.177057Z"}},"outputs":[],"execution_count":null},{"id":"487bac57","cell_type":"code","source":"col_groups_missing_data = get_col_groups_missing_data(train_data)\nprint(f\"The following are all the column groups with rows missing data: {col_groups_missing_data}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:45.179260Z","iopub.execute_input":"2025-08-28T07:14:45.179562Z","iopub.status.idle":"2025-08-28T07:14:46.157582Z","shell.execute_reply.started":"2025-08-28T07:14:45.179534Z","shell.execute_reply":"2025-08-28T07:14:46.156664Z"}},"outputs":[],"execution_count":null},{"id":"7f831f12","cell_type":"markdown","source":"Now that we confirmed these are the only sensor types missing data, let's break down the missing training data by sensor instance.","metadata":{}},{"id":"21b48b1f","cell_type":"code","source":"border = \"-\" * 25\nfor prefix in col_groups_missing_data:\n    print(f\"{border}\\n{prefix}:\\n{border}\")\n    print(train_data[[col for col in train_data.columns if prefix in col]].isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:46.158961Z","iopub.execute_input":"2025-08-28T07:14:46.159572Z","iopub.status.idle":"2025-08-28T07:14:49.867382Z","shell.execute_reply.started":"2025-08-28T07:14:46.159543Z","shell.execute_reply":"2025-08-28T07:14:49.866394Z"}},"outputs":[],"execution_count":null},{"id":"bf8d808b","cell_type":"markdown","source":"It appears the lost IMU orientation data might be uniform across components, which would make sense if each component on the same IMU cut in and out with the sensor as a whole; however it is curious there is no such missing acceleration data. Each thermopile loses data out of synch with each other which makes sense since one sensor failing makes no bearing on whether another one will fail or not. It might be worth noting here that thermopiles 1-4 seem to lose a pretty close ~7k rows of data while thermopile 5 lost almost 5 times as much as the others! It appears for the time-of-flight sensors that the loss may be uniform across the pixels for a given sensor, but can vary widely between sensors. Let's confirm our suspicions.","metadata":{}},{"id":"097207bc","cell_type":"code","source":"uniform_flag = True\nfor prefix in [f\"tof_{i}\" for i in range(1, 6)]:\n    s = train_data[[col for col in train_data.columns if prefix in col]].isnull().sum()\n    if s.nunique() != 1:\n        uniform_flag = False\n\nif uniform_flag:\n    print(\"The number of rows with missing data for all pixels of a given time-of-flight sensor are uniform.\")\n\nfor prefix in [f\"tof_{i}_v0\" for i in range(1, 6)]:\n    print(train_data[[col for col in train_data.columns if prefix in col]].isnull().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:49.868440Z","iopub.execute_input":"2025-08-28T07:14:49.869367Z","iopub.status.idle":"2025-08-28T07:14:51.113359Z","shell.execute_reply.started":"2025-08-28T07:14:49.869344Z","shell.execute_reply":"2025-08-28T07:14:51.112362Z"}},"outputs":[],"execution_count":null},{"id":"f7fd6137","cell_type":"markdown","source":"Interestingly enough, it appears time-of-flight sensors 1-4 lost the same amount of data, while number 5 again lost about 5 times as much. Let's break this down by sequence.","metadata":{}},{"id":"357fbfca","cell_type":"code","source":"missing_summary = train_data.isna().groupby(train_data[\"sequence_id\"]).any().any(axis=1)\n\nsequences_with_missing_df = missing_summary[missing_summary].reset_index()\ntotal_num_seqs = len(train_data.groupby(train_data[\"sequence_id\"]))\nprint(\n    f\"{len(sequences_with_missing_df)} sequences (or {len(sequences_with_missing_df) / total_num_seqs * 100:.2f}% of \"\n    f\"{total_num_seqs} sequences) have missing data.\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:51.114494Z","iopub.execute_input":"2025-08-28T07:14:51.114745Z","iopub.status.idle":"2025-08-28T07:14:53.045527Z","shell.execute_reply.started":"2025-08-28T07:14:51.114726Z","shell.execute_reply":"2025-08-28T07:14:53.044627Z"}},"outputs":[],"execution_count":null},{"id":"e93cbfeb","cell_type":"markdown","source":"Almost 7% of our sequences have some missing data, which isn't too high of a number, but still may be too significant an amount to simply throw away all such sequences from our training data. Let's see how many sequences are missing data for each sensor group, and if these are typically tiny blips missing that are easily replaceable through interpolation or if it is common for columns to be entirely missing data for the sequence.","metadata":{}},{"id":"c72b5591","cell_type":"code","source":"sensors_by_group = train_data.filter(regex=r\"^(rot_w|thm_\\d+|tof_\\d+_v0)$\").columns\nmissing_mask = train_data[sensors_by_group].isna()\n\nnum_seqs_missing_data_per_col = missing_mask.groupby(train_data[\"sequence_id\"]).any().sum()\n\nprint(\"Number of sequences missing data for each column group:\")\nprint(num_seqs_missing_data_per_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.046607Z","iopub.execute_input":"2025-08-28T07:14:53.047318Z","iopub.status.idle":"2025-08-28T07:14:53.143645Z","shell.execute_reply.started":"2025-08-28T07:14:53.047294Z","shell.execute_reply":"2025-08-28T07:14:53.142749Z"}},"outputs":[],"execution_count":null},{"id":"992576e5","cell_type":"code","source":"grouped = missing_mask.groupby(train_data[\"sequence_id\"])\n\nnum_seqs_partially_missing_per_col = (grouped.any() & ~grouped.all()).sum()\n\nprint(\"Number of sequences partially missing data per column:\")\nprint(num_seqs_partially_missing_per_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.144598Z","iopub.execute_input":"2025-08-28T07:14:53.144855Z","iopub.status.idle":"2025-08-28T07:14:53.223799Z","shell.execute_reply.started":"2025-08-28T07:14:53.144826Z","shell.execute_reply":"2025-08-28T07:14:53.223073Z"}},"outputs":[],"execution_count":null},{"id":"01bf433e","cell_type":"markdown","source":"It appears thermopiles 2 and 5 and time-of-flight sensor 5 are only missing data partially for one sequence, respectively. Or in other words, almost all sequences that have _any_ data missing for a column are missing _all_ data for that column. Let's make a heatmap to visualize which sensors gave out for which sequences.","metadata":{}},{"id":"cc5cf729","cell_type":"code","source":"# Map all sequences (x-axis) to whether each sensor (y-axis) is missing any data for that sequence.\nsequence_missing_map = (missing_mask.groupby(train_data[\"sequence_id\"]).any()).T\n\n# Turn the sequence IDs into integers.\nsequence_missing_map.columns = sequence_missing_map.columns.str.extract(r\"SEQ_(\\d+)\")[0].astype(int)\n\nplt.figure(figsize=(12, 6))\n\n# Give each sensor an index and plot the value of that index for each sequence ID for which it has missing data.\nfor i, sensor in enumerate(sequence_missing_map.index):\n    missing_seq_ids = sequence_missing_map.columns[sequence_missing_map.loc[sensor]]\n    plt.scatter(missing_seq_ids, [i] * len(missing_seq_ids), s=1, label=sensor)\n\nplt.yticks(ticks=range(len(sequence_missing_map.index)), labels=sequence_missing_map.index.astype(str).to_list())\nplt.xlabel(\"Sequence ID\")\nplt.ylabel(\"Sensor\")\nplt.title(\"Sensors with Missing Data by Sequence\")\nplt.grid(True, axis=\"x\", linestyle=\":\", alpha=0.3)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.224777Z","iopub.execute_input":"2025-08-28T07:14:53.225110Z","iopub.status.idle":"2025-08-28T07:14:53.623670Z","shell.execute_reply.started":"2025-08-28T07:14:53.225077Z","shell.execute_reply":"2025-08-28T07:14:53.622712Z"}},"outputs":[],"execution_count":null},{"id":"d9b73f79","cell_type":"markdown","source":"From the heatmap above it appears that the sensors cut in and out periodically throughout the sequences, without a clear pattern of sequences with missing data bunching together. We do see what we observed before though with regard to thermopiles 1-4 and time-of-flight sensors 1-4 having similar patterns with regard to the sequences in which they lost data.\n\n## 5. Demographics Data\n\nNow that we have a better idea of what data is missing, let's see what supplemental data we can grab from the demographics table.","metadata":{}},{"id":"d5fdcc5b","cell_type":"code","source":"train_demographics_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.624912Z","iopub.execute_input":"2025-08-28T07:14:53.625270Z","iopub.status.idle":"2025-08-28T07:14:53.636484Z","shell.execute_reply.started":"2025-08-28T07:14:53.625249Z","shell.execute_reply":"2025-08-28T07:14:53.635647Z"}},"outputs":[],"execution_count":null},{"id":"d4941a2e","cell_type":"markdown","source":"Let's check if this table has any nullish entries.","metadata":{}},{"id":"72d9a9e1","cell_type":"code","source":"train_demographics_data.isnull().sum()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.637595Z","iopub.execute_input":"2025-08-28T07:14:53.637826Z","iopub.status.idle":"2025-08-28T07:14:53.654714Z","shell.execute_reply.started":"2025-08-28T07:14:53.637786Z","shell.execute_reply":"2025-08-28T07:14:53.653832Z"}},"outputs":[],"execution_count":null},{"id":"3110e424","cell_type":"markdown","source":"Let's check if there is a bijective map from the training data to the demographics table (i.e. all subjects appear in both tables).","metadata":{}},{"id":"c6631769","cell_type":"code","source":"def get_unq_subject_ids(series: pd.Series) -> set[int]:\n    \"\"\"Returns the collection of unique subject IDs from the relevant column.\n\n    Args:\n        series (pd.Series): The subject IDs column of a data frame.\n\n    Returns:\n        set[int]: The collection of unique subject IDs.\n    \"\"\"\n    return set(series.str.extract(r\"SUBJ_(\\d+)\", expand=False).dropna().astype(int))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.660480Z","iopub.execute_input":"2025-08-28T07:14:53.661340Z","iopub.status.idle":"2025-08-28T07:14:53.670736Z","shell.execute_reply.started":"2025-08-28T07:14:53.661305Z","shell.execute_reply":"2025-08-28T07:14:53.669865Z"}},"outputs":[],"execution_count":null},{"id":"63f266a4","cell_type":"code","source":"ids_train = get_unq_subject_ids(train_data[\"subject\"])\nids_demo = get_unq_subject_ids(train_demographics_data[\"subject\"])\n\nif ids_train == ids_demo:\n    print(\"Unique subjects in the training table have a bijective map to the demographics table!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:53.672179Z","iopub.execute_input":"2025-08-28T07:14:53.672491Z","iopub.status.idle":"2025-08-28T07:14:54.167429Z","shell.execute_reply.started":"2025-08-28T07:14:53.672460Z","shell.execute_reply":"2025-08-28T07:14:54.166556Z"}},"outputs":[],"execution_count":null},{"id":"5fd4667c","cell_type":"markdown","source":"Let's see the statistical breakdown of each feature from the demographics data.","metadata":{}},{"id":"4c9bb294","cell_type":"code","source":"def cv_percent(series: pd.Series) -> float:\n    \"\"\"Returns the coefficient of variation as a percentage for a given feature.\n\n    Args:\n        series (pd.Series): A feature from a data frame.\n\n    Returns:\n        float: The coefficient of variation.\n    \"\"\"\n    return abs(series.std() / series.mean() * 100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:54.168405Z","iopub.execute_input":"2025-08-28T07:14:54.168643Z","iopub.status.idle":"2025-08-28T07:14:54.173484Z","shell.execute_reply.started":"2025-08-28T07:14:54.168624Z","shell.execute_reply":"2025-08-28T07:14:54.172913Z"}},"outputs":[],"execution_count":null},{"id":"560967b9","cell_type":"code","source":"stats_df = train_demographics_data.select_dtypes(include=\"number\").agg([\"mean\", \"std\", \"min\", \"max\", cv_percent])\nstats_df = stats_df.round(2)\n\nprint(stats_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:54.174277Z","iopub.execute_input":"2025-08-28T07:14:54.174548Z","iopub.status.idle":"2025-08-28T07:14:54.203737Z","shell.execute_reply.started":"2025-08-28T07:14:54.174522Z","shell.execute_reply":"2025-08-28T07:14:54.202969Z"}},"outputs":[],"execution_count":null},{"id":"93cdf954","cell_type":"markdown","source":"The statistics above tell us the data is fairly evenly split between children and adults, with about two-thirds of the data taken from people between the ages of 12 and 32. 62% of subjects were male, 88% wore the device on their right hand, and their average dimensions were a height of 5'5\", an arm length of 1'8\", and a forearm length of 10 inches. The latter lengths all seem to have coefficients of variation $\\lesssim$ 10%, so they don't jump out as being of the utmost significance but it is still worth investigating how this might affect classification of the data. One feature that stands out here is the handedness, classifying the same behavior from data for either hand (as it mirrors the orientation) may prove to be difficult, especially since there is a large bias toward right-handed data, so it is worth investigating exactly how this changes any patterns in the data.\n\nLet's also check on how many sequences each subject appeared in.","metadata":{}},{"id":"76bbb9a5","cell_type":"code","source":"subject_sequence_counts = (\n    train_data[[\"subject\", \"sequence_id\"]].drop_duplicates().groupby(\"subject\").size().reset_index(name=\"num_sequences\")\n)\n\nstats_df = subject_sequence_counts[\"num_sequences\"].agg([\"mean\", \"std\", \"min\", \"max\", cv_percent])\nstats_df = stats_df.round(2)\n\nprint(stats_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:54.204779Z","iopub.execute_input":"2025-08-28T07:14:54.205090Z","iopub.status.idle":"2025-08-28T07:14:54.312624Z","shell.execute_reply.started":"2025-08-28T07:14:54.205067Z","shell.execute_reply":"2025-08-28T07:14:54.311871Z"}},"outputs":[],"execution_count":null},{"id":"3837d976","cell_type":"markdown","source":"It looks like each subject participated in about 100 sequences each, so for our approximately 8,000 sequences, there were around 80 total subjects. The competition overview stated that all participants performed all 18 gestures, but they did not do all orientations. Let's take a look at which orientations we are missing for each gesture.\n<a id='orientations_per_gesture_plot'></a>","metadata":{}},{"id":"f1c13855","cell_type":"code","source":"gesture_orientation = train_data[[\"orientation\", \"gesture\"]].drop_duplicates()\ngesture_orientation[\"gesture_code\"] = gesture_orientation[\"gesture\"].astype(\"category\").cat.codes\ngesture_orientation[\"orientation_code\"] = gesture_orientation[\"orientation\"].astype(\"category\").cat.codes\n\nplt.figure(figsize=(12, 6))\n\nplt.scatter(gesture_orientation[\"gesture_code\"], gesture_orientation[\"orientation_code\"], alpha=0.7, edgecolors=\"k\")\n\ngesture_categories = gesture_orientation[\"gesture\"].astype(\"category\").cat.categories\norientation_categories = gesture_orientation[\"orientation\"].astype(\"category\").cat.categories\nplt.xticks(\n    ticks=range(len(gesture_categories)),\n    labels=gesture_categories.astype(str).to_list(),\n    rotation=45,\n    ha=\"right\",\n    fontsize=7,\n)\nplt.yticks(ticks=range(len(orientation_categories)), labels=orientation_categories.astype(str).to_list())\n\nplt.xlabel(\"Gesture\")\nplt.ylabel(\"Orientation\")\nplt.title(\"Orientations per Gesture\")\nplt.grid(True, linestyle=\"--\", alpha=0.4)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:54.313519Z","iopub.execute_input":"2025-08-28T07:14:54.313755Z","iopub.status.idle":"2025-08-28T07:14:54.683610Z","shell.execute_reply.started":"2025-08-28T07:14:54.313738Z","shell.execute_reply":"2025-08-28T07:14:54.682508Z"}},"outputs":[],"execution_count":null},{"id":"5a5773ba","cell_type":"markdown","source":"We can see that we are missing some gesture/orientation combinations, which makes sense because, for example, drinking from a cup while lying on one's back is not a common gesture.\n\n## 6. Time Series Data\n\nNow that we've looked at the demographics data and understood more about what orientations our subjects performed the gestures in, let's finally dive into how these factors affect the time series data! We can start by transforming our sequence counter into normalized time steps to help us compare sequences of different lengths.","metadata":{}},{"id":"626a9f89","cell_type":"code","source":"train_data[\"time_norm\"] = train_data.groupby(\"sequence_id\")[\"sequence_counter\"].transform(lambda x: x / x.max())\ntrain_data[[\"row_id\", \"time_norm\"]].head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:54.684713Z","iopub.execute_input":"2025-08-28T07:14:54.685065Z","iopub.status.idle":"2025-08-28T07:14:56.137675Z","shell.execute_reply.started":"2025-08-28T07:14:54.685044Z","shell.execute_reply":"2025-08-28T07:14:56.136869Z"}},"outputs":[],"execution_count":null},{"id":"8eb5a4d9","cell_type":"markdown","source":"Here we define a function we can use to plot time series data to help us visualize how data for each sensor changes with respect to features like gesture, orientation, phase, and more.","metadata":{}},{"id":"db583229","cell_type":"code","source":"class Phase(Enum):\n    \"\"\"The phase of the sequence.\"\"\"\n\n    TRANSITION = (\"Transition\",)\n    GESTURE = (\"Gesture\",)\n    FULL = (\"Transition\", \"Gesture\")\n\n\nclass PlotTimeseriesProps(BaseModel):\n    \"\"\"Properties for plotting timeseries data for a group of sensors in a given sequence phase.\"\"\"\n\n    train_data: pd.DataFrame\n    train_demographics_data: pd.DataFrame\n    sequence_id: str\n    cols: list[str]\n    phase: Phase\n\n    model_config = ConfigDict(arbitrary_types_allowed=True)\n\n\ndef plot_seq_timeseries(props: PlotTimeseriesProps, show_labels: bool = True, show_stats: bool = True) -> Figure:\n    \"\"\"Plots timeseries data for a group of sensors in a given sequence phase.\n\n    Args:\n        props (PlotTimeseriesProps): Properties for plotting the timeseries data.\n        show_labels (bool, optional): Whether to show labels on the plot. Defaults to True.\n        show_stats (bool, optional): Whether to print statistics for the data. Defaults to True.\n\n    Returns:\n        Figure: The figure object.\n    \"\"\"\n    data = props.train_data[\n        (props.train_data[\"sequence_id\"] == props.sequence_id) & (props.train_data[\"phase\"].isin(props.phase.value))\n    ]\n    subject = str(data.iloc[0][\"subject\"])\n    subj_data = props.train_demographics_data[props.train_demographics_data[\"subject\"] == subject]\n    handedness = subj_data.iloc[0][\"handedness\"]\n\n    title = \"\\n\".join(\n        [\n            f\"Gesture: {data.iloc[0]['gesture']}\",\n            f\"Orientation: {data.iloc[0]['orientation']}\",\n            f\"Phase: {props.phase}\",\n            f\"Handedness: {'Right' if handedness else 'Left'}\",\n            f\"Sequence ID: {props.sequence_id}\",\n        ]\n    )\n\n    if show_stats:\n        stats_df = (\n            data[props.cols]\n            .select_dtypes(include=\"number\")\n            .agg([\"min\", \"max\", \"mean\", \"std\", cv_percent, \"skew\", \"kurtosis\"])\n        )\n        stats_df = stats_df.round(2)\n        print(f\"Statistics:\\n{stats_df}\")\n\n    fig, ax = plt.subplots(figsize=(10, 5))\n\n    for col in props.cols:\n        if show_labels:\n            sns.lineplot(x=data[\"time_norm\"], y=data[col], label=col)\n        else:\n            sns.lineplot(x=data[\"time_norm\"], y=data[col])\n\n    if show_labels:\n        ax.legend()\n\n    if props.phase == Phase.FULL:\n        gesture_start_idx = data[data[\"phase\"] == \"Gesture\"].index[0]\n        gesture_start_time = data[\"time_norm\"].loc[gesture_start_idx]\n        ax.axvline(x=gesture_start_time, color=\"gray\", linestyle=\"--\")\n\n    ax.set_title(title)\n    ax.set_xlabel(\"Normalized Time\")\n    ax.set_ylabel(\"\")\n    ax.grid()\n    return fig","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:56.138770Z","iopub.execute_input":"2025-08-28T07:14:56.139102Z","iopub.status.idle":"2025-08-28T07:14:56.151728Z","shell.execute_reply.started":"2025-08-28T07:14:56.139075Z","shell.execute_reply":"2025-08-28T07:14:56.150970Z"}},"outputs":[],"execution_count":null},{"id":"754ce545","cell_type":"markdown","source":"Let's first find a sequence we can use with the gesture `Above ear - pull hair` and orientation `Seated Straight`. We are starting with this gesture because it provides us the opportunity to visualize a target gesture which occurs in all orientations according to the plot above. ","metadata":{}},{"id":"19c0f3a4","cell_type":"code","source":"gesture = \"Above ear - pull hair\"\norientation = \"Seated Straight\"\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[30]\n)\nsequence_id","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:56.152827Z","iopub.execute_input":"2025-08-28T07:14:56.153141Z","iopub.status.idle":"2025-08-28T07:14:56.279171Z","shell.execute_reply.started":"2025-08-28T07:14:56.153114Z","shell.execute_reply":"2025-08-28T07:14:56.278383Z"}},"outputs":[],"execution_count":null},{"id":"54e91766","cell_type":"markdown","source":"## 6.1. IMU Data\n\nLet's begin by visualizing our inertial measurement unit (IMU) data, since that will always be an available feature for our model. Below we plot the components of the accelerometer's measured acceleration.","metadata":{}},{"id":"2f8cdbb1","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:56.280201Z","iopub.execute_input":"2025-08-28T07:14:56.280773Z","iopub.status.idle":"2025-08-28T07:14:56.641500Z","shell.execute_reply.started":"2025-08-28T07:14:56.280743Z","shell.execute_reply":"2025-08-28T07:14:56.640608Z"}},"outputs":[],"execution_count":null},{"id":"6de40401","cell_type":"markdown","source":"We can see what appears to be the transition from about 0.4 to 0.5 units of our normalized time, followed by the period of rest until about 0.6, followed by what appears to be approximately six repetitions of the gesture. We don't have a lot of information about whether the sampling rate is constant, or what it might be, so we can come to a crude estimate from [the video](https://www.youtube.com/watch?v=qUrwgD4mTt8&list=PL_qYbUpkYMRUJSsRnfl0UpyCCV4RspUmh&index=5).","metadata":{}},{"id":"b93aaeab","cell_type":"code","source":"video_len_s = 5\nnum_gest_in_video = 6\n\ngesture_data = train_data[(train_data[\"sequence_id\"] == sequence_id) & (train_data[\"phase\"] == \"Gesture\")]\nseq_len = len(train_data[train_data[\"sequence_id\"] == sequence_id])\ngesture_len = len(gesture_data)\nvideo_gest_freq = num_gest_in_video / video_len_s\nsample_rate = gesture_len / video_len_s\n\nprint(\n    \"\\n\".join(\n        [\n            f\"Length of video (seconds): {video_len_s}\",\n            f\"Gestures in video: {num_gest_in_video}\",\n            f\"Video gesture frequency (gestures/second): {video_gest_freq}\",\n            f\"Approximate sample rate (samples/second): {sample_rate}\",\n            f\"Total sequence time (seconds): {seq_len / sample_rate:.2f}\",\n        ]\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:56.642582Z","iopub.execute_input":"2025-08-28T07:14:56.642903Z","iopub.status.idle":"2025-08-28T07:14:56.786453Z","shell.execute_reply.started":"2025-08-28T07:14:56.642880Z","shell.execute_reply":"2025-08-28T07:14:56.785504Z"}},"outputs":[],"execution_count":null},{"id":"12869bd9","cell_type":"markdown","source":"In the video, the subject does six repetitions of the gesture in five seconds. If we treat the oscillations we are seeing in the plot above as repetitions of the gesture (i.e. we are assuming the same number of repetitions of the gesture as the video and each trial and none of the oscillations are due to any noise sources or the lack of precision of the instrument), we can crudely approximate the sampling rate to be 6.2 Hz for a total sequence time of 11.94 seconds. [The IMU](https://fcon_1000.projects.nitrc.org/indi/cmi_healthy_brain_network/Competitions/Helios2025/IMU_Sensor.pdf) is capable of 1 kHz sampling, so this frequency might be tailored by CMI to specifically filter out noise. According to [[Vernooij _et al._, 2014](https://doi.org/10.1152/jn.00519.2014)], the human physiological mechanical resonance for fingers is between 20 and 25 Hz. The oscillations in the data (1.2 Hz at a $\\approx 5$ Hz sample rate) then seem to be significant ($<3.1$ Hz according to the [Nyquist-Shannon sampling theorem](https://en.wikipedia.org/wiki/Nyquist%E2%80%93Shannon_sampling_theorem)) sources of a subject's mechanical motion and not noise due to the typical resonance of the wrist.\n\nThese accelerations do not oscillate about zero, which would be more representative of the mechanical motion of the subject, because of the influence of the force of gravity in which objects in free fall accelerate towards the earth's surface at $9.81\\frac{m}{s^2}$. Let's look at the total magnitude of acceleration here and then subtract the component due to gravity to get a better sense of the motion the subject is doing.","metadata":{}},{"id":"76610410","cell_type":"code","source":"acc_data = cast(pd.DataFrame, train_data[train_data[\"sequence_id\"] == sequence_id]).copy(deep=True)\nacc_data[\"acc_mag\"] = np.sqrt(acc_data[\"acc_x\"] ** 2 + acc_data[\"acc_y\"] ** 2 + acc_data[\"acc_z\"] ** 2)\n\nprops = PlotTimeseriesProps(\n    train_data=acc_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_mag\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\n\nacc_grav = 9.81\nacc_data[\"acc_mag_no_grav\"] = acc_data[\"acc_mag\"] - acc_grav\n\nprops = PlotTimeseriesProps(\n    train_data=acc_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_mag_no_grav\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:56.787454Z","iopub.execute_input":"2025-08-28T07:14:56.788048Z","iopub.status.idle":"2025-08-28T07:14:57.272171Z","shell.execute_reply.started":"2025-08-28T07:14:56.788020Z","shell.execute_reply":"2025-08-28T07:14:57.271126Z"}},"outputs":[],"execution_count":null},{"id":"3ba171e9","cell_type":"markdown","source":"The acceleration magnitude plot above compensating for gravity more accurately represents the motion through space done by the subject. From the components of acceleration plot, it appears the subject starts at rest, then rotates into position (since the resting periods before the transition and before the gesture fall into equilibriums at significantly different accelerations). For this reason (the subject is rotating causing gravity to shift each component's (x, y, or z) acceleration measurement differently), we need to incorporate the IMU's orientation quaternion (fused from gyroscope, accelerometer, and magnetometer data) to help provide orientation details and filter gravity from the components of acceleration. Let's first take a look at what this orientation data looks like.","metadata":{}},{"id":"fbfe31af","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=acc_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:57.273100Z","iopub.execute_input":"2025-08-28T07:14:57.273336Z","iopub.status.idle":"2025-08-28T07:14:57.578056Z","shell.execute_reply.started":"2025-08-28T07:14:57.273316Z","shell.execute_reply":"2025-08-28T07:14:57.577207Z"}},"outputs":[],"execution_count":null},{"id":"dd54b33b","cell_type":"markdown","source":"This looks more or less as predicted based on the acceleration data: starting at rest, rotating into position, resting, then oscillating in orientation as the subject repeats the gesture. Let's take advantage of this data to plot the components of acceleration, removing the effects due to gravity. Unfortunately, the sensor drift (often corrected by GPS in aerospace contexts) may make the velocity and displacement plots via numerical methods methods (like cumulative trapezoidal integration) inaccurate. \n\nFor coordinate transformations of rotating rigid bodies, rotation matrices, Euler angles, and quaternions are often used. Rotation matrices require more memory than a 4-vector quaternion, and this kind of compactness can be critical in real-time embedded software applications on aerospace vehicles (or small, low-power, wrist-worn devices), where IMUs are often used. Euler angles provide more intuition for rotations in three-dimensional space than quaternions, but they run into the issue of gimbal lock, which is the loss of a rotational degree of freedom when a vehicle is in a steep ascent or descent such that two of its axes (pitch/yaw/roll) are now parallel [[\"Quaternions and spatial rotation\", 2025](https://en.wikipedia.org/wiki/Quaternions_and_spatial_rotation)]. For these reasons, quaternions are often used to represent the orientation of an aerospace vehicle from IMU data. Below we can use them to transform the coordinates of the components of linear acceleration in the body-fixed frame of the wrist-worn device to the space-fixed frame of the laboratory in which the experiment was performed, such that the acceleration due to gravity remains constant along the z-axis, allowing us to filter it from our components.","metadata":{}},{"id":"8e7f501c","cell_type":"code","source":"seq_data = cast(pd.DataFrame, train_data[train_data[\"sequence_id\"] == sequence_id]).copy(deep=True)\nqw, qx, qy, qz = [seq_data[col] for col in [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"]]\nacc_x, acc_y, acc_z = [seq_data[col] for col in [\"acc_x\", \"acc_y\", \"acc_z\"]]\nquaternions = np.stack([qx, qy, qz, qw], axis=1)\nacc_body_fixed = np.stack([acc_x, acc_y, acc_z], axis=1)\nr = R.from_quat(quaternions)\nacc_space_fixed = r.apply(acc_body_fixed)\ngravity = np.array([0, 0, 9.81])\nacc_no_gravity = acc_space_fixed - gravity\nseq_data[[\"acc_x_nograv\", \"acc_y_nograv\", \"acc_z_nograv\"]] = acc_no_gravity\n\nprops = PlotTimeseriesProps(\n    train_data=seq_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x_nograv\", \"acc_y_nograv\", \"acc_z_nograv\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:57.579043Z","iopub.execute_input":"2025-08-28T07:14:57.579286Z","iopub.status.idle":"2025-08-28T07:14:57.901020Z","shell.execute_reply.started":"2025-08-28T07:14:57.579266Z","shell.execute_reply":"2025-08-28T07:14:57.900132Z"}},"outputs":[],"execution_count":null},{"id":"130b5832","cell_type":"markdown","source":"Now we can see the components of the subject's acceleration in the space-fixed frame: at rest, accelerating the device to the ear, resting again, then pulling their hair a number of times. I wonder if providing the model with the space-fixed components of acceleration could be a useful feature. \n\nLet's look at the data for another sequence that has all the same properties so we can examine the similarities we want the model to pick up on.","metadata":{}},{"id":"22200e6e","cell_type":"code","source":"gesture = \"Above ear - pull hair\"\norientation = \"Seated Straight\"\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\n\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[4]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:57.902101Z","iopub.execute_input":"2025-08-28T07:14:57.902437Z","iopub.status.idle":"2025-08-28T07:14:58.847786Z","shell.execute_reply.started":"2025-08-28T07:14:57.902407Z","shell.execute_reply":"2025-08-28T07:14:58.847006Z"}},"outputs":[],"execution_count":null},{"id":"8d579093","cell_type":"markdown","source":"For a right-handed subject seated straight, the sequences have similarities, albeit the oscillations in the first sequence for the z-component seem more pronounced, and the transition period is horizontally shifted due to the subject moving at a different time when instructed after their sequences started. During the transition period for both sequences, the x and z-components go from positive to negative, and the y-component does the opposite. Let's compare the orientation data between these sequences.","metadata":{}},{"id":"3b0e72c1","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\n\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[4]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_001225\",\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:58.848938Z","iopub.execute_input":"2025-08-28T07:14:58.849256Z","iopub.status.idle":"2025-08-28T07:14:59.687923Z","shell.execute_reply.started":"2025-08-28T07:14:58.849230Z","shell.execute_reply":"2025-08-28T07:14:59.687113Z"}},"outputs":[],"execution_count":null},{"id":"62162fce","cell_type":"markdown","source":"We see a similar pattern here. Let's now look at how handedness changes our data.","metadata":{}},{"id":"b3485432","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\n\ngesture = \"Above ear - pull hair\"\norientation = \"Seated Straight\"\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[0]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:14:59.689045Z","iopub.execute_input":"2025-08-28T07:14:59.689579Z","iopub.status.idle":"2025-08-28T07:15:00.489576Z","shell.execute_reply.started":"2025-08-28T07:14:59.689552Z","shell.execute_reply":"2025-08-28T07:15:00.488707Z"}},"outputs":[],"execution_count":null},{"id":"082f3867","cell_type":"markdown","source":"We can see that changing the handedness of this sequence essentially rotated the x-component of the acceleration about a horizontal axis between its resting acceleration measurements before and after the transition. For the orientation data, the change in handedness has appeared to vertically negatively shift the z-component of the rotation and has approximately reflected the rotation's y-component about the time axis. Let's move on to see how the orientation affects our data.","metadata":{}},{"id":"4e0df67d","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\n\ngesture = \"Above ear - pull hair\"\norientation = \"Lie on Side - Non Dominant\"\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[0]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:00.490638Z","iopub.execute_input":"2025-08-28T07:15:00.490938Z","iopub.status.idle":"2025-08-28T07:15:01.288926Z","shell.execute_reply.started":"2025-08-28T07:15:00.490915Z","shell.execute_reply":"2025-08-28T07:15:01.288010Z"}},"outputs":[],"execution_count":null},{"id":"3b0004ee","cell_type":"markdown","source":"Changing from the subject seated straight to lying on their non-dominant side, the largest difference that jumps out to me is there isn't as exaggerated a negative drop in the x-component during the transition. Let's look at the rotation data as well.","metadata":{}},{"id":"1d6db62d","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:01.290033Z","iopub.execute_input":"2025-08-28T07:15:01.290570Z","iopub.status.idle":"2025-08-28T07:15:02.061111Z","shell.execute_reply.started":"2025-08-28T07:15:01.290548Z","shell.execute_reply":"2025-08-28T07:15:02.060226Z"}},"outputs":[],"execution_count":null},{"id":"a16d99e5","cell_type":"markdown","source":"It appears in this orientation, the y and z-components of the rotation quaternion have undergone similar reflections about a horizontal axis. These vertical translations and reflections about horizontal axes seem to be common patterns for how handedness and orientation might translate the IMU data. We may want to think about how to make our model recognize these as being the same features, whether we can filter for this or use this to augment the data with more sequences with similar translations.\n\nLet's see if we can observe a similar pattern with the other orientations.","metadata":{}},{"id":"455d79fa","cell_type":"code","source":"train_data[\"orientation\"].drop_duplicates()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:02.062036Z","iopub.execute_input":"2025-08-28T07:15:02.062291Z","iopub.status.idle":"2025-08-28T07:15:02.093756Z","shell.execute_reply.started":"2025-08-28T07:15:02.062271Z","shell.execute_reply":"2025-08-28T07:15:02.092822Z"}},"outputs":[],"execution_count":null},{"id":"b4d07cd1","cell_type":"code","source":"gesture = \"Above ear - pull hair\"\norientation = \"Seated Lean Non Dom - FACE DOWN\"\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[0]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:02.094713Z","iopub.execute_input":"2025-08-28T07:15:02.095014Z","iopub.status.idle":"2025-08-28T07:15:02.905632Z","shell.execute_reply.started":"2025-08-28T07:15:02.094994Z","shell.execute_reply":"2025-08-28T07:15:02.904782Z"}},"outputs":[],"execution_count":null},{"id":"89543f47","cell_type":"markdown","source":"Curiously, the acceleration plot for sitting leaning forward looks very similar to that of lying on their side. These seem like quite different orientations so the reason for this isn't obvious to me at this moment. The orientation data, however, looks different from the other orientations. Behaviorally (ignoring vertical translations), y-component of the rotation is reflected about a horizontal axis compared to sitting straight and the z-component is similarly reflected when compared to lying on their side. It makes some intuitive sense that there would be a reflection of one rotation axis once the subject has been rotated into a different orientation. Now let's see what happens when the subject is lying on their back.","metadata":{}},{"id":"f43ffbc5","cell_type":"code","source":"gesture = \"Above ear - pull hair\"\norientation = \"Lie on Back\"\nsequence_id = (\n    train_data[(train_data[\"gesture\"] == gesture) & (train_data[\"orientation\"] == orientation)][\"sequence_id\"]\n    .drop_duplicates()\n    .iloc[0]\n)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=sequence_id,\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_stats=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:02.906739Z","iopub.execute_input":"2025-08-28T07:15:02.907417Z","iopub.status.idle":"2025-08-28T07:15:03.718541Z","shell.execute_reply.started":"2025-08-28T07:15:02.907388Z","shell.execute_reply":"2025-08-28T07:15:03.717647Z"}},"outputs":[],"execution_count":null},{"id":"4343df0c","cell_type":"markdown","source":"For the acceleration, it appears that when seated straight, the acceleration primarily oscillates in the z-direction, whereas when lying on their back it appears to be primarily in the y-direction. Again for the rotation, it appears behaviorally like it is one component's horizontal reflection away from the other orientations. For example, the z-component is flipped compared to sitting and the y-component is flipped compared to lying on their side. It appears both the y and z-components are flipped compared to sitting leaning forward. \n\nSince we're trying to distinguish between sequences that might involve multiple repetitions of gestures, Fourier analysis of these sequences could prove to be useful. Let's start by defining a function that will plot the discrete Fourier transform of a collection of columns from our data. Since the repetitions start during the gesture, followed by a resting period, the Fourier transforms of the gestures will likely be more informative.","metadata":{}},{"id":"98ed72f9","cell_type":"code","source":"def plot_fft(props: PlotTimeseriesProps, show_labels: bool = True) -> Figure:\n    \"\"\"Plots the 1-D discrete Fourier Transform for a given sequence's collection of features.\n\n    Args:\n        props (PlotTimeseriesProps): Properties for plotting the timeseries data.\n        show_labels (bool, optional): Whether to show labels on the plot. Defaults to True.\n\n    Returns:\n        Figure: The figure object.\n    \"\"\"\n    data = props.train_data[\n        (props.train_data[\"sequence_id\"] == props.sequence_id) & (props.train_data[\"phase\"].isin(props.phase.value))\n    ]\n    subject = str(data.iloc[0][\"subject\"])\n    subj_data = props.train_demographics_data[props.train_demographics_data[\"subject\"] == subject]\n    handedness = subj_data.iloc[0][\"handedness\"]\n\n    title = \"\\n\".join(\n        [\n            f\"Gesture: {data.iloc[0]['gesture']}\",\n            f\"Orientation: {data.iloc[0]['orientation']}\",\n            f\"Phase: {props.phase}\",\n            f\"Handedness: {'Right' if handedness else 'Left'}\",\n            f\"Sequence ID: {props.sequence_id}\",\n        ]\n    )\n\n    fig, ax = plt.subplots(figsize=(10, 5))\n    peak_ticks: list[float] = []\n\n    for col in props.cols:\n        signal = data[col].astype(float).to_numpy()\n\n        # Remove the DC component.\n        signal = signal - np.mean(signal)\n        fft_vals = np.abs(np.array(rfft(signal)))\n        fft_vals[0] = 0\n\n        fft_freqs = rfftfreq(len(signal), d=1)\n        peak_freq = fft_freqs[np.argmax(fft_vals)]\n        peak_ticks.append(peak_freq)\n\n        plt.axvline(x=peak_freq, linestyle=\"--\", color=\"gray\", alpha=0.6)\n        sns.lineplot(x=fft_freqs, y=fft_vals, label=f\"fft_{col}\" if show_labels else None, ax=ax)\n\n    all_ticks = sorted({*ax.get_xticks(), *peak_ticks})\n    tick_labels = [f\"{tick:.2f}\" for tick in all_ticks]\n    ax.set_xticks(all_ticks)\n    ax.set_xticklabels(tick_labels, rotation=45)\n    ax.set_xlim(0.0, 0.5)\n\n    if show_labels:\n        ax.legend()\n\n    ax.set_title(title)\n    ax.set_xlabel(\"Frequency (Cycles per Sequence Count)\")\n    ax.set_ylabel(\"Magnitude\")\n    ax.grid(True)\n    plt.tight_layout()\n    return fig","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:03.719933Z","iopub.execute_input":"2025-08-28T07:15:03.720182Z","iopub.status.idle":"2025-08-28T07:15:03.908156Z","shell.execute_reply.started":"2025-08-28T07:15:03.720162Z","shell.execute_reply":"2025-08-28T07:15:03.907219Z"}},"outputs":[],"execution_count":null},{"id":"b9dd4f08","cell_type":"markdown","source":"Let's take a look at what our Fourier transform data looks like.","metadata":{}},{"id":"0eee3b28","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.GESTURE,\n)\nplot_seq_timeseries(props, show_stats=False)\nplot_fft(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:03.909151Z","iopub.execute_input":"2025-08-28T07:15:03.909548Z","iopub.status.idle":"2025-08-28T07:15:04.626130Z","shell.execute_reply.started":"2025-08-28T07:15:03.909518Z","shell.execute_reply":"2025-08-28T07:15:04.625213Z"}},"outputs":[],"execution_count":null},{"id":"8506ecad","cell_type":"code","source":"sample_freq = 6.2\nx_peak = 0.03\nz_peak = 0.23\ngest_len = len(train_data[(train_data[\"sequence_id\"] == \"SEQ_011548\") & (train_data[\"phase\"] == \"Gesture\")])\n\nprint(\n    \"\\n\".join(\n        [\n            f\"acc_x peak frequency (Hz): {x_peak * sample_freq:.2f}\",\n            f\"acc_z peak frequency (Hz): {z_peak * sample_freq:.2f}\",\n            f\"acc_x cycles: {x_peak * gest_len:.2f}\",\n            f\"acc_z cycles: {z_peak * gest_len}\",\n        ]\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:04.627281Z","iopub.execute_input":"2025-08-28T07:15:04.628084Z","iopub.status.idle":"2025-08-28T07:15:04.722883Z","shell.execute_reply.started":"2025-08-28T07:15:04.628059Z","shell.execute_reply":"2025-08-28T07:15:04.722120Z"}},"outputs":[],"execution_count":null},{"id":"2c577a93","cell_type":"markdown","source":"We can see that the x-component of acceleration peaks strongly at 0.03 cycles per sample, and the z-component peaks at 0.23 cycles per sample. Our crude approximation above said the sampling rate was 6.2 samples per second, which makes these approximately 0.19 Hz and 1.43 Hz, respectively. We can see the x-peak corresponds to just under one cycle over the full gesture, which doesn't represent the mechanical motion of the subject we're interested in. The 0.23 frequency, however, corresponds to a believable number of repetitions of the gesture (approximately 6 like in the video), which we can see fairly clearly in the plot of the acceleration above it. This tells me that frequencies above a certain threshold could be useful for our model to help predict gestures and frequencies below such a threshold should be filtered out.\n\nLet's see how this pattern holds for the Fourier transform of another similar sequence.","metadata":{}},{"id":"f3303d64","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_001225\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.GESTURE,\n)\nplot_seq_timeseries(props, show_stats=False)\nplot_fft(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:04.724056Z","iopub.execute_input":"2025-08-28T07:15:04.724397Z","iopub.status.idle":"2025-08-28T07:15:05.435325Z","shell.execute_reply.started":"2025-08-28T07:15:04.724368Z","shell.execute_reply":"2025-08-28T07:15:05.434324Z"}},"outputs":[],"execution_count":null},{"id":"053f0556","cell_type":"markdown","source":"Looking at the acceleration plot, it is harder to tell what is happening as the amplitude of the gesture isn't as large. We can see that the Fourier transform picks up the 0.03 frequency again, making a strong case for filtering this out. It does pick up the local maxima of the z-component at 0.26 which is close to what it was before, making me think the FFT is better at picking up the gesture in the acceleration data than my eyes, therefore making a strong case for using it as a feature. Let's see if the rotation plot shows a similar story.","metadata":{}},{"id":"8a4edf5e","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n    phase=Phase.GESTURE,\n)\nplot_seq_timeseries(props, show_stats=False)\nplot_fft(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:05.436337Z","iopub.execute_input":"2025-08-28T07:15:05.436638Z","iopub.status.idle":"2025-08-28T07:15:06.186283Z","shell.execute_reply.started":"2025-08-28T07:15:05.436603Z","shell.execute_reply":"2025-08-28T07:15:06.185330Z"}},"outputs":[],"execution_count":null},{"id":"93863ef0","cell_type":"markdown","source":"We see a very similar pattern in the rotation data as well. Let's see how a different orientation might affect this.","metadata":{}},{"id":"a9745c35","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_000092\",\n    cols=[\"acc_x\", \"acc_y\", \"acc_z\"],\n    phase=Phase.GESTURE,\n)\nplot_seq_timeseries(props, show_stats=False)\nplot_fft(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:06.187508Z","iopub.execute_input":"2025-08-28T07:15:06.187856Z","iopub.status.idle":"2025-08-28T07:15:06.908389Z","shell.execute_reply.started":"2025-08-28T07:15:06.187826Z","shell.execute_reply":"2025-08-28T07:15:06.907399Z"}},"outputs":[],"execution_count":null},{"id":"0a550e63","cell_type":"markdown","source":"We can see that in the different orientation, we have a different meaningful component (y instead of z) spiking, but it is near the other sequences' significant spikes (0.19 vs. 0.23), making the case that the FT might be useful in helping us choose our gesture regardless of orientation. We have taken an in-depth look at the IMU data for one specific target gesture, but we will want to employ the same tools to investigate the plots of the IMU data for all the other gestures below.","metadata":{}},{"id":"8ac165c6","cell_type":"markdown","source":"## 6.2 Thermopile and Time-of-Flight Sensor Data\n\nWe have taken a detailed look at the IMU data, which was important since that will be all the data we have to rely on for 50% of our predictions. Now let's take a look at the other sensor data and see what clues we can gleam for engineering features for our model.","metadata":{}},{"id":"caa6fe08","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[f\"thm_{i}\" for i in range(1, 6)],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[f\"thm_{i}\" for i in range(1, 6)],\n    phase=Phase.GESTURE,\n)\nplot_fft(props)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:06.909575Z","iopub.execute_input":"2025-08-28T07:15:06.909917Z","iopub.status.idle":"2025-08-28T07:15:07.733723Z","shell.execute_reply.started":"2025-08-28T07:15:06.909888Z","shell.execute_reply":"2025-08-28T07:15:07.732791Z"}},"outputs":[],"execution_count":null},{"id":"ab7fa597","cell_type":"markdown","source":"For the thermopiles, we see a very similar pattern compared to the IMU data above; however, it is more intuitive as it doesn't need to be adjusted for rotation - we can see the temperature increase and decrease based on the proximity to the subject's body. This could make it quite an effective feature at predicting a BFRB solely because this makes it a good measurement of proximity specifically to the warmth of a human body. We see here a very similar pattern in the FT plot as the accelerometer. The combination of these two datasets could be powerful because, although our model might predict a BFRB based on acceleration, it might make a strong case for not being a BFRB if our temperature does not similarly oscillate (since it would then not be close to the subject's body). \n\nLet's now try to get a general sense of the shape of the data from all the pixels of a time-of-flight sensor.","metadata":{}},{"id":"9b0f6dda","cell_type":"code","source":"props = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[f\"tof_1_v{i}\" for i in range(64)],\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_labels=False, show_stats=False)\n\nprops = PlotTimeseriesProps(\n    train_data=train_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=[f\"tof_1_v{i}\" for i in range(64)],\n    phase=Phase.GESTURE,\n)\nplot_fft(props, show_labels=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:07.734730Z","iopub.execute_input":"2025-08-28T07:15:07.735069Z","iopub.status.idle":"2025-08-28T07:15:10.320033Z","shell.execute_reply.started":"2025-08-28T07:15:07.735046Z","shell.execute_reply":"2025-08-28T07:15:10.318975Z"}},"outputs":[],"execution_count":null},{"id":"82fe3311","cell_type":"markdown","source":"This data presents an interesting problem. Upon visual inspection, I see what looks like a collimated beam of oscillations in several of the pixels lining up with the gesture, which could make it and its FT another useful feature. However, there seems to be a significant number of places where there was no signal reflection, causing the value to jump to -1 which makes the FT plot noisy and messes with the frequency. That said, in the FT plot we do see some signals peaking in the right place, albeit surrounded by a lot of noise. Let's see if applying something as simple as dropping the -1 rows helps clean up the noise as a baseline for filtering.","metadata":{}},{"id":"c66e5a4c","cell_type":"code","source":"tof_cols = [f\"tof_1_v{i}\" for i in range(64)]\ntof_data = train_data[train_data[\"sequence_id\"] == \"SEQ_011548\"].copy(deep=True)\ntof_data.replace(-1, np.nan, inplace=True)\n\n# Filter out sensors that never picked up any data.\ntof_cols = [col for col in tof_cols if not tof_data[col].isna().all()]\n\nprops = PlotTimeseriesProps(\n    train_data=tof_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=tof_cols,\n    phase=Phase.FULL,\n)\nplot_seq_timeseries(props, show_labels=False, show_stats=False)\n\nprops = PlotTimeseriesProps(\n    train_data=tof_data,\n    train_demographics_data=train_demographics_data,\n    sequence_id=\"SEQ_011548\",\n    cols=tof_cols,\n    phase=Phase.GESTURE,\n)\nplot_fft(props, show_labels=False)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:10.321073Z","iopub.execute_input":"2025-08-28T07:15:10.321347Z","iopub.status.idle":"2025-08-28T07:15:12.466096Z","shell.execute_reply.started":"2025-08-28T07:15:10.321326Z","shell.execute_reply":"2025-08-28T07:15:12.464883Z"}},"outputs":[],"execution_count":null},{"id":"b10bbccb","cell_type":"markdown","source":"This simple filtering alone seems to have made a significant difference in cleaning the data and filtering out the noise, enhancing the peak of the gesture we're intending to see.\n\nWe've spent a lot of time visualizing the data for one gesture, let's now generate data for all gestures so we can compare them. We will choose the sitting gesture by default since that is the most often included orientation across all the gestures. We use the sitting leaning forward orientation in the cases we cannot use the sitting orientation for as fair a comparison as possible. Please feel free to continue to explore by using this notebook to generate plots of all combinations of orientations and gestures of interest!","metadata":{}},{"id":"7c7c1b4c","cell_type":"code","source":"if EXPORT_PLOTS_TO_PDF:\n    cols_groups = [\n        [\"acc_x\", \"acc_y\", \"acc_z\"],\n        [\"rot_w\", \"rot_x\", \"rot_y\", \"rot_z\"],\n        [f\"thm_{i}\" for i in range(1, 6)],\n        *[[f\"tof_{j}_v{i}\" for i in range(64)] for j in range(1, 6)],\n    ]\n\n    orientations = [\n        \"Seated Straight\",\n        \"Seated Lean Non Dom - FACE DOWN\",\n    ]\n\n    for gesture in train_data[\"gesture\"].drop_duplicates():\n        sequence_id = None\n\n        for orientation in orientations:\n            filtered = train_data[(train_data[\"gesture\"] == str(gesture)) & (train_data[\"orientation\"] == orientation)][\n                \"sequence_id\"\n            ].drop_duplicates()\n\n            if not filtered.empty:\n                sequence_id = filtered.iloc[0]\n                break\n\n        if sequence_id is None:\n            raise RuntimeError(\n                f\"Could not find a sequence with gesture {gesture} and any of the following orientations {orientations}.\"\n            )\n\n        gesture_str = str(gesture).replace(\"- \", \"\").replace(\" \", \"_\").replace(\"/\", \"_\").lower()\n        pdf_path = PLOT_DIR.joinpath(f\"{gesture_str}.pdf\")\n\n        with PdfPages(pdf_path) as pdf:\n            for cols in cols_groups:\n                is_tof = any(\"tof_\" in col for col in cols)\n\n                props = PlotTimeseriesProps(\n                    train_data=train_data,\n                    train_demographics_data=train_demographics_data,\n                    sequence_id=sequence_id,\n                    cols=cols,\n                    phase=Phase.FULL,\n                )\n                ts_plot = plot_seq_timeseries(props, show_stats=False, show_labels=not is_tof)\n                pdf.savefig(ts_plot)\n                plt.close(ts_plot)\n\n                props = PlotTimeseriesProps(\n                    train_data=train_data,\n                    train_demographics_data=train_demographics_data,\n                    sequence_id=sequence_id,\n                    cols=cols,\n                    phase=Phase.GESTURE,\n                )\n                fft_plot = plot_fft(props, show_labels=not is_tof)\n                pdf.savefig(fft_plot)\n                plt.close(fft_plot)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-28T07:15:12.467309Z","iopub.execute_input":"2025-08-28T07:15:12.467577Z","iopub.status.idle":"2025-08-28T07:15:12.478192Z","shell.execute_reply.started":"2025-08-28T07:15:12.467556Z","shell.execute_reply":"2025-08-28T07:15:12.477186Z"}},"outputs":[],"execution_count":null},{"id":"a05fadb1","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}