{"metadata":{"kernelspec":{"display_name":".venv","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\n\ndata_dir = Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:05.678209Z","iopub.execute_input":"2025-06-04T14:30:05.678529Z","iopub.status.idle":"2025-06-04T14:30:05.687563Z","shell.execute_reply.started":"2025-06-04T14:30:05.678506Z","shell.execute_reply":"2025-06-04T14:30:05.686181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!ls -l {data_dir}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:05.688963Z","iopub.execute_input":"2025-06-04T14:30:05.689398Z","iopub.status.idle":"2025-06-04T14:30:05.837456Z","shell.execute_reply.started":"2025-06-04T14:30:05.689374Z","shell.execute_reply":"2025-06-04T14:30:05.836478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"NUMERIC_COLUMNS = [\n    \"acc_x\",\n    \"acc_y\",\n    \"acc_z\",\n    \"rot_w\",\n    \"rot_x\",\n    \"rot_y\",\n    \"rot_z\",\n    *[f\"thm_{i}\" for i in range(1, 6)],\n    *[f\"tof_{i}_v{j}\" for i in range(1, 6) for j in range(64)],\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:05.838547Z","iopub.execute_input":"2025-06-04T14:30:05.838808Z","iopub.status.idle":"2025-06-04T14:30:05.844609Z","shell.execute_reply.started":"2025-06-04T14:30:05.838779Z","shell.execute_reply":"2025-06-04T14:30:05.843671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import polars as pl\n\ntrain_df = pl.read_csv(data_dir / \"train.csv\").with_columns(\n    pl.col(c).replace(-1, None) for c in NUMERIC_COLUMNS if c.startswith(\"tof_\")\n)\ntrain_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:05.846609Z","iopub.execute_input":"2025-06-04T14:30:05.847284Z","iopub.status.idle":"2025-06-04T14:30:21.128050Z","shell.execute_reply.started":"2025-06-04T14:30:05.847226Z","shell.execute_reply":"2025-06-04T14:30:21.127174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(train_df.columns)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:21.129061Z","iopub.execute_input":"2025-06-04T14:30:21.129387Z","iopub.status.idle":"2025-06-04T14:30:21.134177Z","shell.execute_reply.started":"2025-06-04T14:30:21.129360Z","shell.execute_reply":"2025-06-04T14:30:21.133377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"subject\"].n_unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:21.135051Z","iopub.execute_input":"2025-06-04T14:30:21.135334Z","iopub.status.idle":"2025-06-04T14:30:21.173892Z","shell.execute_reply.started":"2025-06-04T14:30:21.135314Z","shell.execute_reply":"2025-06-04T14:30:21.173102Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"sequence_id\"].n_unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:21.174911Z","iopub.execute_input":"2025-06-04T14:30:21.175206Z","iopub.status.idle":"2025-06-04T14:30:21.189924Z","shell.execute_reply.started":"2025-06-04T14:30:21.175176Z","shell.execute_reply":"2025-06-04T14:30:21.189119Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"gesture\"].n_unique()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:21.190726Z","iopub.execute_input":"2025-06-04T14:30:21.191048Z","iopub.status.idle":"2025-06-04T14:30:21.208693Z","shell.execute_reply.started":"2025-06-04T14:30:21.191026Z","shell.execute_reply":"2025-06-04T14:30:21.207976Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.group_by(\"subject\").agg(\n    pl.col(\"sequence_id\").n_unique().alias(\"unique_sequences\")\n).sort(\"unique_sequences\", descending=True).to_pandas().plot(\n    kind=\"bar\",\n    x=\"subject\",\n    y=\"unique_sequences\",\n    title=\"Number of Unique Sequences per Subject\",\n    xlabel=\"Subject\",\n    ylabel=\"Unique Sequences\",\n    figsize=(16, 3),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:21.209591Z","iopub.execute_input":"2025-06-04T14:30:21.209817Z","iopub.status.idle":"2025-06-04T14:30:23.676012Z","shell.execute_reply.started":"2025-06-04T14:30:21.209793Z","shell.execute_reply":"2025-06-04T14:30:23.675052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.group_by(\"sequence_id\").agg(\n    pl.col(\"sequence_counter\").n_unique().alias(\"unique_counters\")\n).sort(\"unique_counters\", descending=True).to_pandas().plot(\n    kind=\"hist\", bins=100,\n    title=\"Distribution of Unique Sequence Counters\",\n    xlabel=\"Unique Sequence Counters\",\n    ylabel=\"Frequency\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:23.679066Z","iopub.execute_input":"2025-06-04T14:30:23.679457Z","iopub.status.idle":"2025-06-04T14:30:24.412999Z","shell.execute_reply.started":"2025-06-04T14:30:23.679434Z","shell.execute_reply":"2025-06-04T14:30:24.411941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.group_by(\"phase\").agg(\n    pl.col(\"sequence_counter\").n_unique().alias(\"unique_counters\")\n).sort(\"unique_counters\", descending=True).to_pandas().plot(\n    kind=\"bar\",\n    x=\"phase\",\n    y=\"unique_counters\",\n    title=\"Number of Unique Sequence Counters per Phase\",\n    xlabel=\"Phase\",\n    ylabel=\"Unique Sequence Counters\",\n    figsize=(8, 4),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:24.413941Z","iopub.execute_input":"2025-06-04T14:30:24.414268Z","iopub.status.idle":"2025-06-04T14:30:24.930454Z","shell.execute_reply.started":"2025-06-04T14:30:24.414217Z","shell.execute_reply":"2025-06-04T14:30:24.929620Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"GESTURES = sorted(train_df[\"gesture\"].unique().to_list())\nGESTURES","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:24.931386Z","iopub.execute_input":"2025-06-04T14:30:24.931664Z","iopub.status.idle":"2025-06-04T14:30:24.950045Z","shell.execute_reply.started":"2025-06-04T14:30:24.931628Z","shell.execute_reply":"2025-06-04T14:30:24.949168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"PHASES = sorted(train_df[\"phase\"].unique().to_list())\nPHASES","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:24.951002Z","iopub.execute_input":"2025-06-04T14:30:24.951352Z","iopub.status.idle":"2025-06-04T14:30:24.966812Z","shell.execute_reply.started":"2025-06-04T14:30:24.951324Z","shell.execute_reply":"2025-06-04T14:30:24.965888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BEHAVIORS = sorted(train_df[\"behavior\"].unique().to_list())\nBEHAVIORS","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:24.967710Z","iopub.execute_input":"2025-06-04T14:30:24.967932Z","iopub.status.idle":"2025-06-04T14:30:24.984874Z","shell.execute_reply.started":"2025-06-04T14:30:24.967915Z","shell.execute_reply":"2025-06-04T14:30:24.984029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ORIENTATIONS = sorted(train_df[\"orientation\"].unique().to_list())\nORIENTATIONS","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:24.985903Z","iopub.execute_input":"2025-06-04T14:30:24.986182Z","iopub.status.idle":"2025-06-04T14:30:25.001954Z","shell.execute_reply.started":"2025-06-04T14:30:24.986163Z","shell.execute_reply":"2025-06-04T14:30:25.001305Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.group_by(\"sequence_id\").agg(pl.col(\"sequence_type\").first()).group_by(\n    \"sequence_type\"\n).agg(pl.len().alias(\"count\")).sort(\"count\", descending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:25.002994Z","iopub.execute_input":"2025-06-04T14:30:25.003298Z","iopub.status.idle":"2025-06-04T14:30:25.315906Z","shell.execute_reply.started":"2025-06-04T14:30:25.003272Z","shell.execute_reply":"2025-06-04T14:30:25.315117Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols = train_df.columns\ndf = train_df.group_by(\"sequence_id\").agg(\n    *[pl.all().n_unique()] + [pl.col(\"sequence_counter\").len().alias(\"seq_len\")]\n)\ndisplay(df)\n\nper_sequence_cols = []\nfor col  in cols:\n    if col not in [\"sequence_id\", \"sequence_counter\"]:\n        if (df[col] == 1).all():\n            per_sequence_cols.append(col)\n\nprint(\"Columns with only one unique value per sequence:\")\nfor col in per_sequence_cols:\n    print(f\"- {col}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:25.316628Z","iopub.execute_input":"2025-06-04T14:30:25.316873Z","iopub.status.idle":"2025-06-04T14:30:30.192168Z","shell.execute_reply.started":"2025-06-04T14:30:25.316856Z","shell.execute_reply":"2025-06-04T14:30:30.191285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def sanitize_col_name(s):\n    return s.lower().replace(\" \", \"_\")\n\n\nsequence_meta_df = (\n    train_df.group_by(\"sequence_id\", maintain_order=True)\n    .agg(\n        *[\n            pl.col(c).first()\n            for c in [\n                \"sequence_type\",\n                \"subject\",\n                \"orientation\",\n                \"gesture\",\n            ]\n        ]\n        + [\n            pl.col(\"phase\").n_unique().alias(\"diversity_phase\"),\n            pl.col(\"behavior\").n_unique().alias(\"diversity_behavior\"),\n        ]\n        + [\n            pl.col(\"sequence_counter\").len().alias(\"seq_len\"),\n            *[\n                (pl.col(\"phase\") == p)\n                .sum()\n                .alias(f\"count_phase_{sanitize_col_name(p)}\")\n                for p in PHASES\n            ],\n            *[\n                (pl.col(\"behavior\") == b)\n                .sum()\n                .alias(f\"count_behavior_{sanitize_col_name(b)}\")\n                for b in BEHAVIORS\n            ],\n        ]\n    )\n    .with_columns(\n        [\n            *[\n                (\n                    pl.col(f\"count_phase_{sanitize_col_name(p)}\") / pl.col(\"seq_len\")\n                ).alias(f\"ratio_phase_{sanitize_col_name(p)}\")\n                for p in PHASES\n            ],\n            *[\n                (\n                    pl.col(f\"count_behavior_{sanitize_col_name(b)}\") / pl.col(\"seq_len\")\n                ).alias(f\"ratio_behavior_{sanitize_col_name(b)}\")\n                for b in BEHAVIORS\n            ],\n        ]\n    )\n)\nprint(sequence_meta_df.height)\nsequence_meta_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.193192Z","iopub.execute_input":"2025-06-04T14:30:30.193494Z","iopub.status.idle":"2025-06-04T14:30:30.525011Z","shell.execute_reply.started":"2025-06-04T14:30:30.193465Z","shell.execute_reply":"2025-06-04T14:30:30.524146Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.525970Z","iopub.execute_input":"2025-06-04T14:30:30.526803Z","iopub.status.idle":"2025-06-04T14:30:30.532018Z","shell.execute_reply.started":"2025-06-04T14:30:30.526772Z","shell.execute_reply":"2025-06-04T14:30:30.531212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.filter(pl.col(\"ratio_behavior_moves_hand_to_target_location\") == 0).height","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.532751Z","iopub.execute_input":"2025-06-04T14:30:30.532970Z","iopub.status.idle":"2025-06-04T14:30:30.556291Z","shell.execute_reply.started":"2025-06-04T14:30:30.532952Z","shell.execute_reply":"2025-06-04T14:30:30.555484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.filter(pl.col(\"ratio_behavior_relaxes_and_moves_hand_to_target_location\") == 0).height","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.557420Z","iopub.execute_input":"2025-06-04T14:30:30.557725Z","iopub.status.idle":"2025-06-04T14:30:30.574255Z","shell.execute_reply.started":"2025-06-04T14:30:30.557698Z","shell.execute_reply":"2025-06-04T14:30:30.573312Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.filter(\n    (pl.col(\"ratio_behavior_moves_hand_to_target_location\") == 0)\n    & (pl.col(\"ratio_behavior_relaxes_and_moves_hand_to_target_location\") == 0)\n).height","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.575433Z","iopub.execute_input":"2025-06-04T14:30:30.575786Z","iopub.status.idle":"2025-06-04T14:30:30.593046Z","shell.execute_reply.started":"2025-06-04T14:30:30.575751Z","shell.execute_reply":"2025-06-04T14:30:30.592297Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"- behavior `moves_hand_to_target_location` and `relaxes_and_moves_hand_to_target_location` is mutually exclusive\n    - 両者を区別するのは何か？","metadata":{}},{"cell_type":"code","source":"target_sequence_meta_df = sequence_meta_df.filter(pl.col(\"sequence_type\") == \"Target\")\nnon_target_sequence_meta_df = sequence_meta_df.filter(\n    pl.col(\"sequence_type\") == \"Non-Target\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.594020Z","iopub.execute_input":"2025-06-04T14:30:30.594354Z","iopub.status.idle":"2025-06-04T14:30:30.613105Z","shell.execute_reply.started":"2025-06-04T14:30:30.594325Z","shell.execute_reply":"2025-06-04T14:30:30.611858Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n_, ax = plt.subplots()\ntarget_sequence_meta_df[\"gesture\"].value_counts().sort(\"count\", descending=True).to_pandas().plot(\n    kind=\"barh\",\n    x=\"gesture\",\n    y=\"count\",\n    title=\"Gesture Counts (Target)\",\n    xlabel=\"Gesture\",\n    ylabel=\"Count\",\n    figsize=(8, 4),\n    ax=ax,\n    color=\"blue\",\n    alpha=0.7,\n    label=\"Target\",\n)\n_, ax = plt.subplots()\nnon_target_sequence_meta_df[\"gesture\"].value_counts().sort(\"count\", descending=True).to_pandas().plot(\n    kind=\"barh\",\n    x=\"gesture\",\n    y=\"count\",\n    title=\"Gesture Counts (Non-Target)\",\n    xlabel=\"Gesture\",\n    ylabel=\"Count\",\n    figsize=(8, 4),\n    ax=ax,\n    color=\"orange\",\n    alpha=0.7,\n    label=\"Non-Target\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:30.614324Z","iopub.execute_input":"2025-06-04T14:30:30.614619Z","iopub.status.idle":"2025-06-04T14:30:31.099532Z","shell.execute_reply.started":"2025-06-04T14:30:30.614592Z","shell.execute_reply":"2025-06-04T14:30:31.098399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df[\"diversity_behavior\"].value_counts().sort(\"count\", descending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.100558Z","iopub.execute_input":"2025-06-04T14:30:31.100866Z","iopub.status.idle":"2025-06-04T14:30:31.107670Z","shell.execute_reply.started":"2025-06-04T14:30:31.100847Z","shell.execute_reply":"2025-06-04T14:30:31.106968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.filter(pl.col(\"diversity_behavior\") == 2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.108607Z","iopub.execute_input":"2025-06-04T14:30:31.108813Z","iopub.status.idle":"2025-06-04T14:30:31.125580Z","shell.execute_reply.started":"2025-06-04T14:30:31.108798Z","shell.execute_reply":"2025-06-04T14:30:31.124712Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df[\"diversity_phase\"].value_counts().sort(\"count\", descending=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.126492Z","iopub.execute_input":"2025-06-04T14:30:31.126748Z","iopub.status.idle":"2025-06-04T14:30:31.142194Z","shell.execute_reply.started":"2025-06-04T14:30:31.126729Z","shell.execute_reply":"2025-06-04T14:30:31.141272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.filter(pl.col(\"diversity_phase\") == 1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.146080Z","iopub.execute_input":"2025-06-04T14:30:31.146364Z","iopub.status.idle":"2025-06-04T14:30:31.159191Z","shell.execute_reply.started":"2025-06-04T14:30:31.146341Z","shell.execute_reply":"2025-06-04T14:30:31.158451Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"(sequence_meta_df[\"count_phase_gesture\"] / sequence_meta_df[\"seq_len\"]).to_pandas().plot(\n    kind=\"hist\", bins=100,\n    title=\"Distribution of Gesture Phase Count per Sequence Length\",\n    xlabel=\"Gesture Phase Ratio\",\n    ylabel=\"Frequency\",\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.160324Z","iopub.execute_input":"2025-06-04T14:30:31.160638Z","iopub.status.idle":"2025-06-04T14:30:31.488178Z","shell.execute_reply.started":"2025-06-04T14:30:31.160608Z","shell.execute_reply":"2025-06-04T14:30:31.487383Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sequence_meta_df.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.488992Z","iopub.execute_input":"2025-06-04T14:30:31.489263Z","iopub.status.idle":"2025-06-04T14:30:31.494919Z","shell.execute_reply.started":"2025-06-04T14:30:31.489216Z","shell.execute_reply":"2025-06-04T14:30:31.494087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nfor b in BEHAVIORS:\n    _, ax = plt.subplots()\n    for seq_type, df in zip(\n        [\"Target\", \"Non-Target\"], [target_sequence_meta_df, non_target_sequence_meta_df]\n    ):\n        (df[f\"count_behavior_{sanitize_col_name(b)}\"] / df[\"seq_len\"]).to_pandas().plot(\n            kind=\"hist\",\n            bins=100,\n            title=f\"Distribution of `{b}`\",\n            xlabel=\"Behavior Ratio\",\n            ylabel=\"Frequency\",\n            alpha=0.5,\n            ax=ax,\n            xlim=(0, 1),\n            label=seq_type,\n            color=\"blue\" if seq_type == \"Target\" else \"orange\",\n            legend=True,\n        )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:31.495882Z","iopub.execute_input":"2025-06-04T14:30:31.496170Z","iopub.status.idle":"2025-06-04T14:30:33.670087Z","shell.execute_reply.started":"2025-06-04T14:30:31.496144Z","shell.execute_reply":"2025-06-04T14:30:33.669266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nb1, b2 = \"Moves Hand to Target Location\", \"Relaxes and Moves Hand to Target Location\"\n_, ax = plt.subplots()\nfor seq_type, df in zip(\n    [\"Target\", \"Non-Target\"], [target_sequence_meta_df, non_target_sequence_meta_df]\n):\n    (\n        (\n            df[f\"count_behavior_{sanitize_col_name(b1)}\"]\n            + df[f\"count_behavior_{sanitize_col_name(b2)}\"]\n        )\n        / df[\"seq_len\"]\n    ).to_pandas().plot(\n        kind=\"hist\",\n        bins=100,\n        title=f\"Distribution of `{b1}` and `{b2}` Combined\",\n        xlabel=\"Behavior Ratio\",\n        ylabel=\"Frequency\",\n        alpha=0.5,\n        ax=ax,\n        xlim=(0, 1),\n        label=seq_type,\n        color=\"blue\" if seq_type == \"Target\" else \"orange\",\n        legend=True,\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:33.671048Z","iopub.execute_input":"2025-06-04T14:30:33.671543Z","iopub.status.idle":"2025-06-04T14:30:34.220763Z","shell.execute_reply.started":"2025-06-04T14:30:33.671511Z","shell.execute_reply":"2025-06-04T14:30:34.219963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"behavior\"].unique().to_list()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:34.221549Z","iopub.execute_input":"2025-06-04T14:30:34.221825Z","iopub.status.idle":"2025-06-04T14:30:34.238070Z","shell.execute_reply.started":"2025-06-04T14:30:34.221795Z","shell.execute_reply":"2025-06-04T14:30:34.237387Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Visualize Sample Sequence","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom scipy.spatial.transform import Rotation as R\nfrom mpl_toolkits.mplot3d import Axes3D\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.mplot3d.art3d import Line3DCollection\nfrom matplotlib.collections import LineCollection\nfrom matplotlib import cm\nfrom matplotlib.colors import Normalize\nfrom scipy.ndimage import gaussian_filter1d\nfrom scipy.spatial.transform import Rotation as R, Slerp\n\n\ndef moving_average(data, window_size=5):\n    return np.convolve(data, np.ones(window_size) / window_size, mode=\"same\")\n\n\ndef visualize_tof(tof1, tof2, tof3, tof4, tof5, x_gesture, behavior_dict):\n    for name, x in zip(\n        [\"tof1\", \"tof2\", \"tof3\", \"tof4\", \"tof5\"], [tof1, tof2, tof3, tof4, tof5]\n    ):\n        x = x.copy()\n        x = x.reshape(-1, 64)\n        fig, ax = plt.subplots(figsize=(8, 3))\n        x_max = np.nanmax(x)\n        x_min = np.nanmin(x)\n\n        colors = plt.cm.magma(np.linspace(0, 1, x.shape[1]))\n        for i in range(x.shape[1]):\n            ax.plot(\n                x[:, i],\n                color=colors[i],\n                alpha=0.3,\n                linewidth=0.5,\n            )\n\n        ax.plot(\n            x_gesture * (x_max - x_min) + x_min,\n            color=\"red\",\n            alpha=0.7,\n            linewidth=2,\n            label=\"Gesture Phase\",\n        )\n\n        colors = [\"blue\", \"orange\", \"green\", \"purple\", \"brown\"]\n        for i, b in enumerate(BEHAVIORS):\n            ax.plot(\n                behavior_dict[b] * (x_max - x_min) + x_min,\n                label=f\"Behavior: {b}\",\n                linewidth=1,\n                alpha=0.7,\n                color=colors[i % len(colors)],\n                linestyle=\"--\",\n            )\n        ax.set(\n            title=f\"{name}\",\n            xlabel=\"Sequence Counter\",\n            ylabel=\"Value\",\n        )\n        ax.legend(bbox_to_anchor=(1.05, 1), loc=\"upper left\")\n        plt.show()\n\n\ndef visualize_sensor(acc, acc_world, rot, thm, x_gesture, behavior_dict):\n    for name, x, channel_names in zip(\n        [\"acc_sensor\", \"acc_world\", \"rot\", \"trm\"],\n        [acc, acc_world, rot, thm],\n        [\n            [\"acc_x\", \"acc_y\", \"acc_z\"],\n            [\"acc_x\", \"acc_y\", \"acc_z\"],\n            [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"],\n            [f\"thm_{i}\" for i in range(1, 6)],\n        ],\n    ):\n        x = x.copy()\n        fig, ax = plt.subplots(figsize=(8, 3))\n        x_max = np.max(x)\n        x_min = np.min(x)\n\n        colors = [\"blue\", \"orange\", \"green\", \"purple\", \"brown\"]\n        for i in range(x.shape[1]):\n            ax.plot(\n                x[:, i],\n                color=colors[i % len(colors)],\n                alpha=0.7,\n                linewidth=0.5,\n                label=channel_names[i],\n            )\n\n        ax.plot(\n            x_gesture * (x_max - x_min) + x_min,\n            color=\"red\",\n            alpha=0.7,\n            linewidth=2,\n            label=\"Gesture Phase\",\n        )\n        for i, b in enumerate(BEHAVIORS):\n            ax.plot(\n                behavior_dict[b] * (x_max - x_min) + x_min,\n                label=f\"Behavior: {b}\",\n                linewidth=1,\n                alpha=0.7,\n                color=colors[i % len(colors)],\n                linestyle=\"--\",\n            )\n        ax.set(\n            title=f\"{name}\",\n            xlabel=\"Sequence Counter\",\n            ylabel=\"Value\",\n        )\n        ax.legend(bbox_to_anchor=(1.05, 1), loc=\"upper left\")\n        plt.show()\n\n\ndef plot_in_3d(x, title=\"3D Acceleration Trajectory\", unit=\"m/s²\"):\n    n_samples = x.shape[0]\n\n    # 線分生成（2点ごとのつなぎ）\n    points = x.reshape(-1, 1, 3)\n    segments = np.concatenate([points[:-1], points[1:]], axis=1)\n\n    # カラーマップ用の正規化\n    norm = Normalize(vmin=0, vmax=n_samples - 1)\n    colors = cm.viridis(norm(np.arange(n_samples - 1)))\n\n    # 3D線分に色をつける\n    lc = Line3DCollection(segments, colors=colors, linewidth=1)\n\n    # 描画\n    fig = plt.figure(figsize=(8, 6))\n    ax = fig.add_subplot(111, projection=\"3d\")\n    ax.add_collection3d(lc)\n    # 軸設定\n    ax.set_xlim(np.nanmin(x[:, 0]), np.nanmax(x[:, 0]))\n    ax.set_ylim(np.nanmin(x[:, 1]), np.nanmax(x[:, 1]))\n    ax.set_zlim(np.nanmin(x[:, 2]), np.nanmax(x[:, 2]))\n    ax.set_xlabel(f\"X ({unit})\")\n    ax.set_ylabel(f\"Y ({unit})\")\n    ax.set_zlabel(f\"Z ({unit}\")\n    ax.set_aspect(\"equal\")\n    ax.set_title(title)\n\n    # カラーバー追加\n    sm = plt.cm.ScalarMappable(cmap=cm.viridis, norm=norm)\n    sm.set_array([])  # ダミーデータ（matplotlib仕様）\n    cbar = fig.colorbar(sm, ax=ax, pad=0.1)\n    cbar.set_label(\"Time step index\")\n\n    plt.show()\n\n\ndef plot_transparent_sphere(\n    ax, center=(0, 0, 0), radius=1.0, color=\"cyan\", alpha=0.3, resolution=100\n):\n    \"\"\"\n    半透明な球体を 3D プロットに描画する\n\n    Parameters:\n        ax        : matplotlib の 3D Axes オブジェクト\n        center    : 球体の中心座標 (x, y, z)\n        radius    : 球体の半径\n        color     : 球体の色\n        alpha     : 透明度（0.0〜1.0）\n        resolution: 緯度・経度の分割数（細かさ）\n    \"\"\"\n    u = np.linspace(0, 2 * np.pi, resolution)\n    v = np.linspace(0, np.pi, resolution)\n\n    x = radius * np.outer(np.cos(u), np.sin(v)) + center[0]\n    y = radius * np.outer(np.sin(u), np.sin(v)) + center[1]\n    z = radius * np.outer(np.ones_like(u), np.cos(v)) + center[2]\n\n    ax.plot_surface(x, y, z, color=color, alpha=alpha, edgecolor=\"none\")\n\n\ndef plot_in_3d_with_vector(x, title=\"3D Acceleration Trajectory\", step=10):\n    n_samples = x.shape[0]\n\n    # 線分生成（2点ごとのつなぎ）\n    points = x.reshape(-1, 1, 3)\n    segments = np.concatenate([points[:-1], points[1:]], axis=1)\n\n    # カラーマップ用の正規化\n    norm = Normalize(vmin=0, vmax=n_samples - 1)\n    colors = cm.viridis(norm(np.arange(n_samples - 1)))\n\n    # 3D線分に色をつける\n    lc = Line3DCollection(segments, colors=colors, linewidth=1)\n\n    # 描画\n    fig = plt.figure(figsize=(8, 6))\n    ax = fig.add_subplot(111, projection=\"3d\")\n    ax.add_collection3d(lc)\n\n    for i in range(0, n_samples, step):\n        # origin of arrow (e.g., can use acc[i] instead of (0,0,0) if preferred)\n        origin = np.array([0, 0, 0])\n        direction = x[i]\n        ax.quiver(*origin, *direction, normalize=False, color=\"red\", alpha=0.6)\n\n    ax.scatter([0], [0], [0], color=\"red\", s=50, label=\"Origin\")\n    plot_transparent_sphere(ax, center=(0, 0, 0), radius=1.0, color=\"cyan\", alpha=0.1)\n\n    # 軸設定\n    ax.set(\n        xlim=(-1, 1),\n        ylim=(-1, 1),\n        zlim=(-1, 1),\n        xlabel=\"X\",\n        ylabel=\"Y\",\n        zlabel=\"Z\",\n        aspect=\"equal\",\n        title=title,\n    )\n\n    # カラーバー追加\n    sm = plt.cm.ScalarMappable(cmap=cm.viridis, norm=norm)\n    sm.set_array([])  # ダミーデータ（matplotlib仕様）\n    cbar = fig.colorbar(sm, ax=ax, pad=0.1)\n    cbar.set_label(\"Time step index\")\n\n    plt.show()\n\n\ndef compute_acc_world_with_slerp(acc_sensor, rot, delay_step=0, fill=np.nan):\n    \"\"\"\n    Convert sensor acceleration to world coordinates, correcting for constant delay.\n\n    Parameters:\n        acc_sensor: (N, 3) array of acceleration in sensor coordinates\n        rot: (M, 4) array of quaternions (x, y, z, w)\n        delay_step: constant offset between acc_sensor and rotation (in steps)\n\n    Returns:\n        acc_world: (N, 3) array of acceleration in world coordinates\n    \"\"\"\n    N = len(acc_sensor)\n    M = len(rot)\n    rot_times = np.arange(M)\n    acc_times = np.arange(N) + delay_step\n\n    # マスクして acc_times が rot_times の範囲内のものだけ残す\n    valid_idx = (acc_times >= rot_times[0]) & (acc_times <= rot_times[-1])\n    acc_times_valid = acc_times[valid_idx]\n    acc_sensor_valid = acc_sensor[valid_idx]\n\n    # 補間と回転適用\n    rot_objs = R.from_quat(rot)  # shape: (M,)\n    slerp = Slerp(rot_times, rot_objs)\n    interp_rots = slerp(acc_times_valid)  # shape: (valid_N,)\n    acc_world_valid = interp_rots.apply(acc_sensor_valid)\n\n    # 出力配列を構築（NaNで初期化し、validな位置に結果を埋める）\n    acc_world = np.full_like(acc_sensor, fill)\n    acc_world[valid_idx] = acc_world_valid\n\n    return acc_world\n\n\ndef visualize_sequence(train_df, sequence_id, delay_step=0):\n    single_sequence = train_df.filter(pl.col(\"sequence_id\") == sequence_id)\n\n    display(\n        single_sequence.select(\n            [\"sequence_id\", \"subject\", \"orientation\", \"gesture\", \"sequence_type\"]\n        )\n        .head(1)\n        .to_pandas()\n        .T\n    )\n\n    tof1 = single_sequence.select(f\"tof_1_v{i}\" for i in range(64)).to_numpy()\n    tof1 = tof1.reshape(-1, 8, 8)\n    tof2 = single_sequence.select(f\"tof_2_v{i}\" for i in range(64)).to_numpy()\n    tof2 = tof2.reshape(-1, 8, 8)\n    tof3 = single_sequence.select(f\"tof_3_v{i}\" for i in range(64)).to_numpy()\n    tof3 = tof3.reshape(-1, 8, 8)\n    tof4 = single_sequence.select(f\"tof_4_v{i}\" for i in range(64)).to_numpy()\n    tof4 = tof4.reshape(-1, 8, 8)\n    tof5 = single_sequence.select(f\"tof_5_v{i}\" for i in range(64)).to_numpy()\n    tof5 = tof5.reshape(-1, 8, 8)\n    acc = single_sequence.select(\"acc_x\", \"acc_y\", \"acc_z\").to_numpy()\n    rot = single_sequence.select(\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\").to_numpy()\n    rot_norm = np.linalg.norm(rot, axis=1, keepdims=True)\n    thm = single_sequence.select(f\"thm_{i}\" for i in range(1, 6)).to_numpy()\n    print(f\"rot_norm: mean={np.nanmean(rot_norm):.3f}, std={np.nanstd(rot_norm):.5f}\")\n\n    try:\n        # convert sensor coord to world coord\n        acc_world = compute_acc_world_with_slerp(acc, rot, delay_step=delay_step)\n    except Exception as e:\n        print(f\"Error converting sensor coordinates to world coordinates: {e}\")\n        acc_world = np.zeros_like(acc)\n\n    try:\n        # convert sensor coord to world coord\n        r = R.from_quat(rot)\n        sensor_z = np.zeros_like(np.ones_like(acc))\n        sensor_z[:, 2] = 1.0  # Assuming Z is the vertical axis\n        world_z = compute_acc_world_with_slerp(sensor_z, rot)\n    except Exception as e:\n        print(f\"Error converting sensor Z coordinates to world coordinates: {e}\")\n        world_z = np.zeros_like(np.ones_like(acc))\n        world_z[:, 2] = 1.0  # Default to Z-axis\n\n    x_gesture = (single_sequence[\"phase\"] == \"Gesture\").to_numpy().astype(np.float32)\n    gestures_start_index = np.where(x_gesture == 1)[0][0]\n    hand_at_target_index = np.where(\n        single_sequence[\"behavior\"] == \"Hand at target location\"\n    )[0][0]\n\n    print(f\"Hand at target index: {hand_at_target_index}\")\n    print(f\"Gesture start index: {gestures_start_index}\")\n    behavior_dict = {}\n\n    for b in BEHAVIORS:\n        behavior_dict[b] = (\n            (single_sequence[\"behavior\"] == b).to_numpy().astype(np.float32)\n        )\n\n    print(f\"{tof1.shape=}\")\n    print(f\"{tof2.shape=}\")\n    print(f\"{tof3.shape=}\")\n    print(f\"{tof4.shape=}\")\n    print(f\"{tof5.shape=}\")\n    print(f\"{acc.shape=}\")\n    print(f\"{rot.shape=}\")\n    print(f\"{thm.shape=}\")\n\n    gravity = np.array([0, 0, 9.81])\n\n    visualize_sensor(acc, acc_world, rot, thm, x_gesture, behavior_dict)\n    plot_in_3d(\n        acc,\n        title=\"Acceleration Trajectory (Sensor Coordinates)\",\n    )\n    plot_in_3d(\n        acc_world - gravity,\n        title=\"Acceleration Trajectory (World Coordinates, Gravity Subtracted)\",\n    )\n    plot_in_3d(\n        (acc_world - gravity)[hand_at_target_index:],\n        title=\"Acceleration Trajectory After Moved to the Target Location (World Coordinates, Gravity Subtracted)\",\n    )\n    plot_in_3d_with_vector(world_z, title=\"Z Trajectory (World Coordinates)\")\n    plot_in_3d_with_vector(\n        world_z[hand_at_target_index:],\n        title=\"Z Trajectory After Moved to the Target Location (World Coordinates)\",\n    )\n    visualize_tof(tof1, tof2, tof3, tof4, tof5, x_gesture, behavior_dict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:34.239202Z","iopub.execute_input":"2025-06-04T14:30:34.239519Z","iopub.status.idle":"2025-06-04T14:30:34.564826Z","shell.execute_reply.started":"2025-06-04T14:30:34.239493Z","shell.execute_reply":"2025-06-04T14:30:34.564184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i, g in enumerate(BEHAVIORS):\n    print(f\"{i}: {g}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:34.565548Z","iopub.execute_input":"2025-06-04T14:30:34.565963Z","iopub.status.idle":"2025-06-04T14:30:34.570760Z","shell.execute_reply.started":"2025-06-04T14:30:34.565934Z","shell.execute_reply":"2025-06-04T14:30:34.569910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gesture_to_type = {\n    d[\"gesture\"]: d[\"sequence_type\"]\n    for d in train_df.group_by(\"gesture\")\n    .agg(pl.col(\"sequence_type\").eq(\"Target\").first())\n    .sort(\"sequence_type\", \"gesture\")\n    .to_dicts()\n}\n\nfor i, (k, v) in enumerate(gesture_to_type.items()):\n    tag = \"🚨\" if v else \"✅\"\n    print(f\"{tag} {i}: {k}: {v}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:41:20.795639Z","iopub.execute_input":"2025-06-04T14:41:20.795940Z","iopub.status.idle":"2025-06-04T14:41:20.855382Z","shell.execute_reply.started":"2025-06-04T14:41:20.795917Z","shell.execute_reply":"2025-06-04T14:41:20.854426Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from IPython.display import Markdown\n\nfor i, (gesture, is_target) in enumerate(gesture_to_type.items()):\n    sequence_id = sequence_meta_df.filter(pl.col(\"gesture\") == gesture).sample(\n        1, seed=2, with_replacement=False\n    )[\"sequence_id\"][0]\n    tag = \"🚨\" if is_target else \"✅\"\n    display(Markdown(f\"### {tag} {i}: {gesture=}, {sequence_id=}\"))\n    visualize_sequence(train_df, sequence_id, delay_step=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:43:17.905614Z","iopub.execute_input":"2025-06-04T14:43:17.905972Z","iopub.status.idle":"2025-06-04T14:44:34.429460Z","shell.execute_reply.started":"2025-06-04T14:43:17.905945Z","shell.execute_reply":"2025-06-04T14:44:34.428520Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Demographics","metadata":{}},{"cell_type":"markdown","source":"* TODO: EDA on demographics","metadata":{}},{"cell_type":"code","source":"import polars as pl\n\ntrain_demo_df = pl.read_csv(data_dir / \"train_demographics.csv\")\ntrain_demo_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-04T14:30:46.914953Z","iopub.execute_input":"2025-06-04T14:30:46.915635Z","iopub.status.idle":"2025-06-04T14:30:46.929057Z","shell.execute_reply.started":"2025-06-04T14:30:46.915599Z","shell.execute_reply":"2025-06-04T14:30:46.928154Z"}},"outputs":[],"execution_count":null}]}