{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport polars as pl\nimport seaborn as sea\nimport polars.selectors as pls\nimport matplotlib.pyplot as plt\nfrom scipy.stats import mannwhitneyu\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\ndef format_pl():\n    \"\"\"FLOAT DISPLAY FORMATTING\"\"\"\n    pl.Config.set_fmt_float(\"mixed\")\n    \"\"\"STRING FORMATTING\"\"\"\n    pl.Config.set_fmt_str_lengths(50)\n    \"\"\"TABLE FORMATTING\"\"\"\n    pl.Config.set_tbl_rows(8)\n    pl.Config.set_tbl_cols(15)\n    pl.Config.set_tbl_width_chars(200)\n    pl.Config.set_tbl_cell_alignment(\"RIGHT\")\n    pl.Config.set_tbl_hide_dtype_separator(True)\n    pl.Config.set_tbl_hide_column_data_types(True)\n\nformat_pl()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:30:59.877107Z","iopub.execute_input":"2025-06-05T01:30:59.877299Z","iopub.status.idle":"2025-06-05T01:31:02.394158Z","shell.execute_reply.started":"2025-06-05T01:30:59.877275Z","shell.execute_reply":"2025-06-05T01:31:02.393482Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Data","metadata":{}},{"cell_type":"code","source":"train = pl.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\n\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:02.395315Z","iopub.execute_input":"2025-06-05T01:31:02.395593Z","iopub.status.idle":"2025-06-05T01:31:14.764731Z","shell.execute_reply.started":"2025-06-05T01:31:02.395570Z","shell.execute_reply":"2025-06-05T01:31:14.764127Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## First Subset","metadata":{}},{"cell_type":"code","source":"subset1_features = [\"sequence_id\", \"sequence_type\", \"sequence_counter\", \"subject\", \"phase\", \"gesture\", \"orientation\", \"behavior\"]\nsub1 = train.select(\n    pl.col(subset1_features)\n)\n\nsub1.head(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:14.765418Z","iopub.execute_input":"2025-06-05T01:31:14.765640Z","iopub.status.idle":"2025-06-05T01:31:14.815515Z","shell.execute_reply.started":"2025-06-05T01:31:14.765620Z","shell.execute_reply":"2025-06-05T01:31:14.814812Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gesture_counts = sub1.group_by(\n    pl.col(\"sequence_type\"), pl.col(\"gesture\")\n).agg(\n    pl.col(\"gesture\").count().alias(\"count\")\n).sort(by=\"count\", descending=True).to_pandas()\n\ngesture_counts[\"pct\"] = 100*gesture_counts[\"count\"]/gesture_counts[\"count\"].sum()\n\ngesture_counts","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:14.816224Z","iopub.execute_input":"2025-06-05T01:31:14.816434Z","iopub.status.idle":"2025-06-05T01:31:15.008869Z","shell.execute_reply.started":"2025-06-05T01:31:14.816415Z","shell.execute_reply":"2025-06-05T01:31:15.008174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 6))\n\nplt.subplot(121)\nsea.barplot(\n    data=gesture_counts,\n    y=\"gesture\",\n    x=\"count\",\n    hue=\"sequence_type\",\n    dodge=False\n)\nplt.xticks(rotation=90)\nplt.title(\"Frequency of Gestures in Data\")\nplt.xlabel(\"Frequency\")\nplt.ylabel(\"\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:15.010808Z","iopub.execute_input":"2025-06-05T01:31:15.011313Z","iopub.status.idle":"2025-06-05T01:31:15.451352Z","shell.execute_reply.started":"2025-06-05T01:31:15.011292Z","shell.execute_reply":"2025-06-05T01:31:15.450469Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsea.barplot(\n    data=gesture_counts,\n    y=\"gesture\",\n    x=\"pct\",\n    hue=\"sequence_type\",\n    dodge=False\n)\nplt.xticks(rotation=90)\nplt.title(\"Percentage of Gestures in Data\")\nplt.xlabel(\"Percentage (%)\")\nplt.ylabel(\"\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:15.452503Z","iopub.execute_input":"2025-06-05T01:31:15.452717Z","iopub.status.idle":"2025-06-05T01:31:15.848182Z","shell.execute_reply.started":"2025-06-05T01:31:15.452699Z","shell.execute_reply":"2025-06-05T01:31:15.847258Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1.group_by(\n    pl.col(\"phase\", \"behavior\")\n).agg(\n    pl.col(\"phase\").count().alias(\"count\")\n).sort(by=\"phase\", descending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:15.849144Z","iopub.execute_input":"2025-06-05T01:31:15.849385Z","iopub.status.idle":"2025-06-05T01:31:15.954950Z","shell.execute_reply.started":"2025-06-05T01:31:15.849362Z","shell.execute_reply":"2025-06-05T01:31:15.954261Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"behavior_seq_table = sub1.group_by(\n    pl.col(\"behavior\", \"sequence_type\")\n).agg(\n    pl.col(\"behavior\").count().alias(\"count\")\n).sort(by=(\"behavior\",\"sequence_type\"), descending=True)\n\n\norientation_seq_table = sub1.group_by(\n    pl.col(\"orientation\", \"sequence_type\")\n).agg(\n    pl.col(\"orientation\").count().alias(\"count\")\n).sort(by=(\"orientation\",\"sequence_type\"), descending=True)\n\n\nphase_seq_table = sub1.group_by(\n    pl.col(\"phase\", \"sequence_type\")\n).agg(\n    pl.col(\"phase\").count().alias(\"count\")\n).sort(by=(\"phase\",\"sequence_type\"), descending=True)\n\nprint(behavior_seq_table)\nprint(orientation_seq_table)\nprint(phase_seq_table)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:15.955778Z","iopub.execute_input":"2025-06-05T01:31:15.956020Z","iopub.status.idle":"2025-06-05T01:31:16.249527Z","shell.execute_reply.started":"2025-06-05T01:31:15.956001Z","shell.execute_reply":"2025-06-05T01:31:16.248846Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"### Graphs table to analyze any relationships with sequence_type","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.250281Z","iopub.execute_input":"2025-06-05T01:31:16.250652Z","iopub.status.idle":"2025-06-05T01:31:16.253436Z","shell.execute_reply.started":"2025-06-05T01:31:16.250635Z","shell.execute_reply":"2025-06-05T01:31:16.252759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"Each subject contribute multiple kinds of sequences that are target/non-target\"\"\"\n\nsub1.group_by(\n    pl.col(\"subject\", \"sequence_id\", \"sequence_type\")\n).agg(\n    pl.col(\"subject\").count().alias(\"count\")\n).sort(\n    by=(\"subject\", \"sequence_type\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.254032Z","iopub.execute_input":"2025-06-05T01:31:16.254306Z","iopub.status.idle":"2025-06-05T01:31:16.359259Z","shell.execute_reply.started":"2025-06-05T01:31:16.254291Z","shell.execute_reply":"2025-06-05T01:31:16.358632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"What is the count of sequences per subject (how many sequences does each subject contributes) ?\n\nMost subjects contributed 102 sequences, with 5 subjects contributing less, but that is not a large outlier issue. \n\"\"\"\n\n## Sorts by subject \nsubject_seq_table = sub1.group_by(\n    pl.col(\"subject\", \"sequence_id\",)\n).agg(\n    pl.col(\"subject\").count().alias(\"count\")\n).sort(\n    by=(\"subject\")\n)\n\n## Counts subject\nseq_per_subject_table = subject_seq_table.group_by(\n    pl.col(\"subject\")\n).agg(\n    pl.col(\"subject\").count().alias(\"seq_per_subject_count\")\n)\n\nseq_per_subject_table.group_by(\n    pl.col(\"seq_per_subject_count\")\n).agg(\n    pl.col(\"seq_per_subject_count\").count().alias(\"count\")\n).sort(by=\"seq_per_subject_count\", descending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.359949Z","iopub.execute_input":"2025-06-05T01:31:16.360148Z","iopub.status.idle":"2025-06-05T01:31:16.447668Z","shell.execute_reply.started":"2025-06-05T01:31:16.360130Z","shell.execute_reply":"2025-06-05T01:31:16.446950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub1.group_by(\n    pl.col(\"subject\", \"sequence_id\",)\n).agg(\n    pl.col(\"subject\").count().alias(\"count\")\n).sort(\n    by=(\"subject\")\n).group_by(\n    pl.col(\"subject\")\n).agg(\n    pl.col(\"subject\").count().alias(\"seq_per_subject_count\")\n).filter(\n    pl.col(\"seq_per_subject_count\")!=102\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.448391Z","iopub.execute_input":"2025-06-05T01:31:16.448641Z","iopub.status.idle":"2025-06-05T01:31:16.546418Z","shell.execute_reply.started":"2025-06-05T01:31:16.448617Z","shell.execute_reply":"2025-06-05T01:31:16.545660Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"Are the length distribution of target sequences different from non-target sequences?\"\"\"\n\nseq_lengths = sub1.select(\n    pl.col(\"sequence_id\", \"sequence_counter\", \"sequence_type\")\n).group_by(\n    pl.col(\"sequence_id\", \"sequence_type\")\n).agg(\n    pl.col(\"sequence_id\").count().alias(\"length\")\n).to_pandas()\n\nsea.histplot(x=np.log10(seq_lengths[\"length\"]), hue=seq_lengths[\"sequence_type\"])\nplt.xlabel(\"Log10 Sequence Length\")\nplt.ylabel(\"Count\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.547304Z","iopub.execute_input":"2025-06-05T01:31:16.547517Z","iopub.status.idle":"2025-06-05T01:31:16.903798Z","shell.execute_reply.started":"2025-06-05T01:31:16.547499Z","shell.execute_reply":"2025-06-05T01:31:16.903168Z"},"_kg_hide-input":false},"outputs":[],"execution_count":null},{"cell_type":"code","source":"target_lengths    = seq_lengths[seq_lengths[\"sequence_type\"] == \"Target\"][\"length\"]\nnontarget_lengths = seq_lengths[seq_lengths[\"sequence_type\"] == \"Non-Target\"][\"length\"]\n\nu_stat, p_value = mannwhitneyu(\n    target_lengths, \n    nontarget_lengths,\n    alternative=\"less\"   ## median target < median nontarget\n)\n\nprint(f\"Ho: Median lengths of target and non-target sequences are not differnet\")\nprint(f\"Ha: Median length of target is less than median length of non-target sequences\")\nprint(f\"U = {u_stat:.3f}, p = {p_value:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.906372Z","iopub.execute_input":"2025-06-05T01:31:16.906559Z","iopub.status.idle":"2025-06-05T01:31:16.917155Z","shell.execute_reply.started":"2025-06-05T01:31:16.906543Z","shell.execute_reply":"2025-06-05T01:31:16.916273Z"},"jupyter":{"source_hidden":true},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sea.kdeplot(x=target_lengths, label=\"Target\", fill=True)\nsea.kdeplot(nontarget_lengths, label=\"Non-Target\", fill=True)\nplt.title(\"Sequence Length Distributions\")\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:16.917881Z","iopub.execute_input":"2025-06-05T01:31:16.918171Z","iopub.status.idle":"2025-06-05T01:31:17.155375Z","shell.execute_reply.started":"2025-06-05T01:31:16.918149Z","shell.execute_reply":"2025-06-05T01:31:17.154387Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"For each subject, what is the ratio of target/non-target sequences?\"\"\"\n\nsub_seq_table = sub1.group_by(\n    pl.col(\"subject\", \"sequence_id\", \"sequence_type\")\n).agg(\n    pl.col(\"subject\").count().alias(\"count\")\n).sort(\n    by=(\"subject\", \"sequence_type\")\n).group_by(\n    pl.col(\"subject\", \"sequence_type\")\n).agg(\n    pl.col(\"subject\").count().alias(\"count\")\n).sort(\n    by=(\"subject\", \"sequence_type\")\n)\n\n\nsub_seq_table = sub_seq_table.pivot(\n    values=\"count\",\n    index=\"subject\",\n    columns=\"sequence_type\"\n).with_columns(\n    (pl.col(\"Target\")/pl.col(\"Non-Target\")).alias(\"target_nonTarget_ratio\")\n)\n\nratio_counts = sub_seq_table.group_by(\n    pl.col(\"target_nonTarget_ratio\")\n).agg(\n    pl.col(\"target_nonTarget_ratio\").count().alias(\"count\")\n).sort(by=\"count\", descending=True)\n\n\noutlier_ratios = sub_seq_table.filter(\n    (pl.col(\"target_nonTarget_ratio\") < 1.68) | \n    (pl.col(\"target_nonTarget_ratio\") > 1.69)\n)\n\n\nprint(sub_seq_table)\nprint(ratio_counts)\nprint(outlier_ratios)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:17.156242Z","iopub.execute_input":"2025-06-05T01:31:17.156486Z","iopub.status.idle":"2025-06-05T01:31:17.261207Z","shell.execute_reply.started":"2025-06-05T01:31:17.156466Z","shell.execute_reply":"2025-06-05T01:31:17.260637Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Second Subset","metadata":{}},{"cell_type":"code","source":"sub2 = train.select(\n    pl.col(\"^acc_.*$\", \"^rot_.*$\", \"^thm_.*$\", \"^tof_.*_v.*$\")\n)\nsub2.head(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:17.261737Z","iopub.execute_input":"2025-06-05T01:31:17.261957Z","iopub.status.idle":"2025-06-05T01:31:17.268373Z","shell.execute_reply.started":"2025-06-05T01:31:17.261938Z","shell.execute_reply":"2025-06-05T01:31:17.267852Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_NA = sub2.select(\n    pl.all().is_null().sum()\n).unpivot(\n    value_name=\"NA_count\",\n    variable_name=\"feature\"\n).filter(\n    pl.col(\"NA_count\")>0\n).sort(\n    by=(\"feature\", \"NA_count\"),\n    descending=True\n).with_columns(\n    (100*pl.col(\"NA_count\")/len(sub2)).alias(\"NA_pct\")\n)\n\nfeatures_NA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:17.269024Z","iopub.execute_input":"2025-06-05T01:31:17.269214Z","iopub.status.idle":"2025-06-05T01:31:17.296375Z","shell.execute_reply.started":"2025-06-05T01:31:17.269200Z","shell.execute_reply":"2025-06-05T01:31:17.295814Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub2.select(\n    pl.all().min()\n).unpivot(\n    value_name=\"min_value\"\n).sort(\n    by=\"min_value\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:17.297169Z","iopub.execute_input":"2025-06-05T01:31:17.297737Z","iopub.status.idle":"2025-06-05T01:31:17.363539Z","shell.execute_reply.started":"2025-06-05T01:31:17.297720Z","shell.execute_reply":"2025-06-05T01:31:17.362927Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub2 = sub2.with_columns(\n    # Acceleration magnitude\n    pl.fold(\n        acc = pl.lit(0),\n        function = lambda acc,x: acc + x**2,\n        exprs = pl.col(\"^acc_.*$\")\n    ).sqrt().alias(\"acc_mag\"),\n\n    # Rotational angle (rad) from neutral position\n    (2*np.arccos(pl.col(\"rot_w\").clip(-1,1))).alias(\"rot_angle\"),\n    \n    # Thermopile mean\n    (pl.fold(\n        acc = pl.lit(0),\n        function = lambda acc,x: acc + x,\n        exprs = pl.col(\"^thm_.*$\")\n     )/5).alias(\"thm_mean\"),\n    \n    # Thermopile std\n    pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.std().alias(\"thm_std\"),\n    \n    # Thermopile range: max - min\n    (pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.max() - \\\n     pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.min()).alias(\"thm_range\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:17.364212Z","iopub.execute_input":"2025-06-05T01:31:17.364438Z","iopub.status.idle":"2025-06-05T01:31:18.928946Z","shell.execute_reply.started":"2025-06-05T01:31:17.364416Z","shell.execute_reply":"2025-06-05T01:31:18.928251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sensors_summaries = sub2.select(\n    pl.col(\"acc_mag\", \"rot_angle\", \"thm_mean\", \"thm_std\", \"thm_range\")\n).drop_nulls().to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:18.929656Z","iopub.execute_input":"2025-06-05T01:31:18.929871Z","iopub.status.idle":"2025-06-05T01:31:18.948422Z","shell.execute_reply.started":"2025-06-05T01:31:18.929854Z","shell.execute_reply":"2025-06-05T01:31:18.947430Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16,6))\n\nplt.subplot(221)\nsea.histplot(x=sensors_summaries[\"acc_mag\"])\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\nplt.title(\"Acceleration Magnitude Distribution\")\n\nplt.subplot(222)\nsea.histplot(x=np.log10(sensors_summaries[\"acc_mag\"]))\nplt.xlabel(\"Log10 Acceleration Magnitude\")\nplt.title(\"Log Acceleration Magnitude Distribution\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:18.949311Z","iopub.execute_input":"2025-06-05T01:31:18.949562Z","iopub.status.idle":"2025-06-05T01:31:26.863283Z","shell.execute_reply.started":"2025-06-05T01:31:18.949525Z","shell.execute_reply":"2025-06-05T01:31:26.862588Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sea.histplot(x=sensors_summaries[\"rot_angle\"]*180/np.pi, kde=True)\nplt.xlabel(\"Rotational Angle (Degrees)\")\nplt.title(\"Rotational Angles of Users\")\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:26.864248Z","iopub.execute_input":"2025-06-05T01:31:26.864452Z","iopub.status.idle":"2025-06-05T01:31:28.725702Z","shell.execute_reply.started":"2025-06-05T01:31:26.864433Z","shell.execute_reply":"2025-06-05T01:31:28.724858Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"k = 20000\nsample_mean = sensors_summaries[\"thm_mean\"].sample(n=k, random_state=42)\nsample_std  = sensors_summaries[\"thm_std\"].sample(n=k, random_state=42)\nx_range = np.arange(0, k)\n\nplt.figure(figsize=(16, 6))\nplt.subplot(121)\nsea.scatterplot(x=x_range,\n                y=sample_mean, \n                alpha=.05)\n\nplt.fill_between(x=x_range,\n                 y1=sample_mean - sample_std,\n                 y2=sample_mean + sample_std,\n                 alpha=1,\n                 label=\"±1 std\")\nplt.title(\"Mean Temperature over 5 Sensors ±1 std\")\nplt.ylabel(\"Mean Temperature over 5 Sensors\")\nplt.xlabel(\"Sample\")\nplt.legend()\n\nplt.subplot(122)\nsea.histplot(x=sensors_summaries[\"thm_range\"])\nplt.title(\"Range of Temperatures over 5 Sensors (Max - Min)\")\nplt.xlabel(\"Temperature Range\")\n\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:28.726619Z","iopub.execute_input":"2025-06-05T01:31:28.726839Z","iopub.status.idle":"2025-06-05T01:31:29.853104Z","shell.execute_reply.started":"2025-06-05T01:31:28.726823Z","shell.execute_reply":"2025-06-05T01:31:29.851946Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Combined Analysis","metadata":{}},{"cell_type":"code","source":"train2 = train.with_columns(\n    # Acceleration magnitude\n    pl.fold(\n        acc = pl.lit(0),\n        function = lambda acc,x: acc + x**2,\n        exprs = pl.col(\"^acc_.*$\")\n    ).sqrt().alias(\"acc_mag\"),\n\n    # Rotational angle (rad) from neutral position\n    (2*np.arccos(pl.col(\"rot_w\").clip(-1,1))).alias(\"rot_angle\"),\n    \n    # Thermopile mean\n    (pl.fold(\n        acc = pl.lit(0),\n        function = lambda acc,x: acc + x,\n        exprs = pl.col(\"^thm_.*$\")\n     )/5).alias(\"thm_mean\"),\n    \n    # Thermopile std\n    pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.std().alias(\"thm_std\"),\n    \n    # Thermopile range: max - min\n    (pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.max() - \\\n     pl.concat_list([\"thm_1\", \"thm_2\", \"thm_3\", \"thm_4\", \"thm_5\"]).list.min()).alias(\"thm_range\")\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:29.853837Z","iopub.execute_input":"2025-06-05T01:31:29.854073Z","iopub.status.idle":"2025-06-05T01:31:31.310313Z","shell.execute_reply.started":"2025-06-05T01:31:29.854056Z","shell.execute_reply":"2025-06-05T01:31:31.309529Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train2.group_by(\"sequence_type\").agg(\n    pl.col(\"acc_mag\").median().alias(\"median_acc_mag\"),\n    pl.col(\"rot_angle\").median().alias(\"median_rot_angle\"),\n    pl.col(\"thm_mean\").median().alias(\"median_thm\"),\n    pl.col(\"thm_std\").median().alias(\"median_thm_std\"),\n    pl.col(\"thm_range\").median().alias(\"median_thm_range\"),\n).sort(by=\"sequence_type\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:31.310966Z","iopub.execute_input":"2025-06-05T01:31:31.311239Z","iopub.status.idle":"2025-06-05T01:31:31.836228Z","shell.execute_reply.started":"2025-06-05T01:31:31.311217Z","shell.execute_reply":"2025-06-05T01:31:31.835532Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train2.group_by(\"sequence_type\").agg(\n    pl.col(\"acc_mag\").mean().alias(\"mean_acc_mag\"),\n    pl.col(\"rot_angle\").mean().alias(\"mean_rot_angle\"),\n    pl.col(\"thm_mean\").mean().alias(\"mean_thm\"),\n    pl.col(\"thm_std\").mean().alias(\"mean_thm_std\"),\n    pl.col(\"thm_range\").mean().alias(\"mean_thm_range\"),\n).sort(by=\"sequence_type\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:31.837002Z","iopub.execute_input":"2025-06-05T01:31:31.837179Z","iopub.status.idle":"2025-06-05T01:31:32.054793Z","shell.execute_reply.started":"2025-06-05T01:31:31.837163Z","shell.execute_reply":"2025-06-05T01:31:32.053859Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train2.group_by(\n    pl.col(\"sequence_type\", \"gesture\")\n).agg(\n    pl.col(\"acc_mag\").median().alias(\"median_acc_mag\"),\n    pl.col(\"rot_angle\").median().alias(\"median_rot_angle\"),\n    pl.col(\"thm_mean\").median().alias(\"median_thm\"),\n    pl.col(\"thm_std\").median().alias(\"median_thm_std\"),\n    pl.col(\"thm_range\").median().alias(\"median_thm_range\"),\n).sort(by=\"sequence_type\").to_pandas()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:32.055726Z","iopub.execute_input":"2025-06-05T01:31:32.055946Z","iopub.status.idle":"2025-06-05T01:31:32.278847Z","shell.execute_reply.started":"2025-06-05T01:31:32.055929Z","shell.execute_reply":"2025-06-05T01:31:32.277962Z"},"_kg_hide-input":false,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"k = 20000\nsequence_data = train2.select(\n    pl.col(\"sequence_type\", \"acc_mag\", \"rot_angle\", \"thm_mean\", \"thm_std\", \"thm_range\")\n).to_pandas()\nsampled_data = sequence_data.sample(n=k, random_state=42)\n\n\n\nplt.figure(figsize=(16, 12))\n\nplt.subplot(3,2,1)\nsea.boxplot(\n    data=sampled_data,\n    x=\"acc_mag\",\n    y=\"sequence_type\",\n)\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\n\nplt.subplot(3,2,2)\nsea.boxplot(\n    data=sampled_data,\n    x=\"rot_angle\",\n    y=\"sequence_type\",\n)\nplt.xlabel(\"Rotational Angle (Radians)\")\n\nplt.subplot(3,2,3)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_mean\",\n    y=\"sequence_type\",\n)\nplt.xlabel(\"Mean Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,4)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_std\",\n    y=\"sequence_type\",\n)\nplt.xlabel(\"SD of Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,5)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_range\",\n    y=\"sequence_type\",\n)\nplt.xlabel(\"Range of Temperature across 5 Sensors (Celsius)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:32.279546Z","iopub.execute_input":"2025-06-05T01:31:32.279736Z","iopub.status.idle":"2025-06-05T01:31:32.888329Z","shell.execute_reply.started":"2025-06-05T01:31:32.279719Z","shell.execute_reply":"2025-06-05T01:31:32.887736Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"k = 20000\n\nphase_orientation_sensors = train2.select(\n    pl.col(\"phase\", \"orientation\", \"behavior\", \"acc_mag\", \"rot_angle\", \"thm_mean\", \"thm_std\", \"thm_range\")\n).to_pandas()\n\nsampled_data = phase_orientation_sensors.sample(n=k, random_state=42)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-06-05T01:31:32.888990Z","iopub.execute_input":"2025-06-05T01:31:32.889183Z","iopub.status.idle":"2025-06-05T01:31:32.986203Z","shell.execute_reply.started":"2025-06-05T01:31:32.889166Z","shell.execute_reply":"2025-06-05T01:31:32.985530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 12))\n\nplt.subplot(3,2,1)\nsea.boxplot(\n    data=sampled_data,\n    x=\"acc_mag\",\n    y=\"phase\",\n)\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\n\nplt.subplot(3,2,2)\nsea.boxplot(\n    data=sampled_data,\n    x=\"rot_angle\",\n    y=\"phase\",\n)\nplt.xlabel(\"Rotational Angle (Radians)\")\n\nplt.subplot(3,2,3)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_mean\",\n    y=\"phase\",\n)\nplt.xlabel(\"Mean Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,4)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_std\",\n    y=\"phase\",\n)\nplt.xlabel(\"SD of Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,5)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_range\",\n    y=\"phase\",\n)\nplt.xlabel(\"Range of Temperature across 5 Sensors (Celsius)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:32.986826Z","iopub.execute_input":"2025-06-05T01:31:32.987044Z","iopub.status.idle":"2025-06-05T01:31:33.498147Z","shell.execute_reply.started":"2025-06-05T01:31:32.987027Z","shell.execute_reply":"2025-06-05T01:31:33.497391Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 12))\n\nplt.subplot(3,2,1)\nsea.boxplot(\n    data=sampled_data,\n    x=\"acc_mag\",\n    y=\"orientation\",\n)\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\n\nplt.subplot(3,2,2)\nsea.boxplot(\n    data=sampled_data,\n    x=\"rot_angle\",\n    y=\"orientation\",\n)\nplt.xlabel(\"Rotational Angle (Radians)\")\n\nplt.subplot(3,2,3)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_mean\",\n    y=\"orientation\",\n)\nplt.xlabel(\"Mean Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,4)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_std\",\n    y=\"orientation\",\n)\nplt.xlabel(\"SD of Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,5)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_range\",\n    y=\"orientation\",\n)\nplt.xlabel(\"Range of Temperature across 5 Sensors (Celsius)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:33.498763Z","iopub.execute_input":"2025-06-05T01:31:33.498939Z","iopub.status.idle":"2025-06-05T01:31:34.121533Z","shell.execute_reply.started":"2025-06-05T01:31:33.498924Z","shell.execute_reply":"2025-06-05T01:31:34.120866Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(16, 12))\n\nplt.subplot(3,2,1)\nsea.boxplot(\n    data=sampled_data,\n    x=\"acc_mag\",\n    y=\"behavior\",\n)\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\n\nplt.subplot(3,2,2)\nsea.boxplot(\n    data=sampled_data,\n    x=\"rot_angle\",\n    y=\"behavior\",\n)\nplt.xlabel(\"Rotational Angle (Radians)\")\n\nplt.subplot(3,2,3)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_mean\",\n    y=\"behavior\",\n)\nplt.xlabel(\"Mean Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,4)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_std\",\n    y=\"behavior\",\n)\nplt.xlabel(\"SD of Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,5)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_range\",\n    y=\"behavior\",\n)\nplt.xlabel(\"Range of Temperature across 5 Sensors (Celsius)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:31:34.122159Z","iopub.execute_input":"2025-06-05T01:31:34.122337Z","iopub.status.idle":"2025-06-05T01:31:35.256533Z","shell.execute_reply.started":"2025-06-05T01:31:34.122313Z","shell.execute_reply":"2025-06-05T01:31:35.255737Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"k = 20000\nsequence_data = train2.select(\n    pl.col(\"gesture\", \"acc_mag\", \"rot_angle\", \"thm_mean\", \"thm_std\", \"thm_range\")\n).to_pandas()\nsampled_data = sequence_data.sample(n=k, random_state=42)\n\n\n\nplt.figure(figsize=(16, 12))\n\nplt.subplot(3,2,1)\nsea.boxplot(\n    data=sampled_data,\n    x=\"acc_mag\",\n    y=\"gesture\",\n)\nplt.xlabel(\"Acceleration Magnitude (m/sec^2)\")\n\nplt.subplot(3,2,2)\nsea.boxplot(\n    data=sampled_data,\n    x=\"rot_angle\",\n    y=\"gesture\",\n)\nplt.xlabel(\"Rotational Angle (Radians)\")\n\nplt.subplot(3,2,3)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_mean\",\n    y=\"gesture\",\n)\nplt.xlabel(\"Mean Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,4)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_std\",\n    y=\"gesture\",\n)\nplt.xlabel(\"SD of Temperature across 5 Sensors (Celsius)\")\n\nplt.subplot(3,2,5)\nsea.boxplot(\n    data=sampled_data,\n    x=\"thm_range\",\n    y=\"gesture\",\n)\nplt.xlabel(\"Range of Temperature across 5 Sensors (Celsius)\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-05T01:34:59.141632Z","iopub.execute_input":"2025-06-05T01:34:59.141873Z","iopub.status.idle":"2025-06-05T01:35:00.414133Z","shell.execute_reply.started":"2025-06-05T01:34:59.141856Z","shell.execute_reply":"2025-06-05T01:35:00.413227Z"},"_kg_hide-input":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null}]}