{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"I have picked up the basic EDA that is considered necessary for the competition.","metadata":{}},{"cell_type":"code","source":"#Train Data\n#/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\n#/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\n\n#Test Data\n#kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\n#kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:02.093363Z","iopub.execute_input":"2025-06-13T01:46:02.093702Z","iopub.status.idle":"2025-06-13T01:46:02.098697Z","shell.execute_reply.started":"2025-06-13T01:46:02.093681Z","shell.execute_reply":"2025-06-13T01:46:02.097430Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Library loading & basic settings","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns  \n\npd.set_option(\"display.max_columns\", 200)\npd.set_option(\"display.width\", 120)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:02.100706Z","iopub.execute_input":"2025-06-13T01:46:02.101934Z","iopub.status.idle":"2025-06-13T01:46:02.118831Z","shell.execute_reply.started":"2025-06-13T01:46:02.101896Z","shell.execute_reply":"2025-06-13T01:46:02.117865Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# ２．Data Lording","metadata":{}},{"cell_type":"code","source":"DATA_DIR = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\n\ntrain       = pd.read_csv(f\"{DATA_DIR}/train.csv\")\ntest        = pd.read_csv(f\"{DATA_DIR}/test.csv\")\ntrain_demo  = pd.read_csv(f\"{DATA_DIR}/train_demographics.csv\")\ntest_demo   = pd.read_csv(f\"{DATA_DIR}/test_demographics.csv\")\n\nprint(f\"train shape: {train.shape}\")\nprint(f\"test  shape: {test.shape}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:02.119737Z","iopub.execute_input":"2025-06-13T01:46:02.120113Z","iopub.status.idle":"2025-06-13T01:46:23.241678Z","shell.execute_reply.started":"2025-06-13T01:46:02.120082Z","shell.execute_reply":"2025-06-13T01:46:23.240694Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3. Check meta information\n# 3-1. Column data type/missing status\n","metadata":{}},{"cell_type":"code","source":"train.info(memory_usage=\"deep\")\ntrain.isna().mean().sort_values(ascending=False).head(20)  # 欠損率 TOP20\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:23.242677Z","iopub.execute_input":"2025-06-13T01:46:23.243000Z","iopub.status.idle":"2025-06-13T01:46:24.783753Z","shell.execute_reply.started":"2025-06-13T01:46:23.242973Z","shell.execute_reply":"2025-06-13T01:46:24.782725Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 3‑2. Duplication/uniquene","metadata":{}},{"cell_type":"code","source":"assert train[\"row_id\"].is_unique\nn_seq_train = train[\"sequence_id\"].nunique()\nn_seq_test  = test[\"sequence_id\"].nunique()\nprint(\"unique sequences – train:\", n_seq_train, \" test:\", n_seq_test)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:24.785992Z","iopub.execute_input":"2025-06-13T01:46:24.786276Z","iopub.status.idle":"2025-06-13T01:46:24.941862Z","shell.execute_reply.started":"2025-06-13T01:46:24.786256Z","shell.execute_reply":"2025-06-13T01:46:24.941075Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Class distribution and sequence length\n# 4‑1. Gesture & sequence_type hist\n","metadata":{}},{"cell_type":"code","source":"seq_level = train.groupby(\"sequence_id\").agg(\n    gesture        = (\"gesture\", \"first\"),\n    sequence_type  = (\"sequence_type\", \"first\"),\n    rows_in_seq    = (\"row_id\", \"count\"),\n    subject        = (\"subject\", \"first\"),\n)\ndisplay(seq_level.head())\n\nfig, ax = plt.subplots(figsize=(10,4))\nseq_level[\"gesture\"].value_counts().plot.bar(ax=ax)\nax.set_title(\"Gesture count (train)\")\nplt.xticks(rotation=45, ha=\"right\")\nplt.tight_layout()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:24.942807Z","iopub.execute_input":"2025-06-13T01:46:24.943144Z","iopub.status.idle":"2025-06-13T01:46:25.867096Z","shell.execute_reply.started":"2025-06-13T01:46:24.943117Z","shell.execute_reply":"2025-06-13T01:46:25.865909Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4‑2. Sequence length distribution","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(6,4))\nsns.histplot(seq_level[\"rows_in_seq\"], bins=50, log_scale=True)\nplt.title(\"Distribution of rows per sequence\")\nplt.xlabel(\"#rows in one sequence\")\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:25.868139Z","iopub.execute_input":"2025-06-13T01:46:25.868425Z","iopub.status.idle":"2025-06-13T01:46:26.231844Z","shell.execute_reply.started":"2025-06-13T01:46:25.868403Z","shell.execute_reply":"2025-06-13T01:46:26.230965Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Defective pattern by sensor","metadata":{}},{"cell_type":"code","source":"sensor_cols = [c for c in train.columns if c.startswith((\"acc_\", \"rot_\", \"thm_\", \"tof_\"))]\nmissing = train[sensor_cols].isna()\n\n# 欠損率\nmiss_rate = missing.mean().sort_values(ascending=False)\nprint(miss_rate.head())\n\n# センサ種別ごと概要\nfor prefix in [\"acc_\", \"rot_\", \"thm_\", \"tof_\"]:\n    cols = [c for c in sensor_cols if c.startswith(prefix)]\n    print(prefix, \"mean missing =\", missing[cols].mean().mean().round(3))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:26.232811Z","iopub.execute_input":"2025-06-13T01:46:26.233146Z","iopub.status.idle":"2025-06-13T01:46:27.364976Z","shell.execute_reply.started":"2025-06-13T01:46:26.233124Z","shell.execute_reply":"2025-06-13T01:46:27.364081Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. IMU statistics and distribution\n\nTip: Make sure that acc_* is around ±9.8m/s^2 and rot_* (quaternion) is within ‑1 to 1.\n\nOutliers can be easily dealt with by clipping or winsorize.","metadata":{}},{"cell_type":"code","source":"imu_cols = [c for c in sensor_cols if c.startswith((\"acc_\", \"rot_\"))]\nimu_stats = train[imu_cols].describe().T[[\"mean\", \"std\", \"min\", \"max\"]]\ndisplay(imu_stats.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:27.365905Z","iopub.execute_input":"2025-06-13T01:46:27.366246Z","iopub.status.idle":"2025-06-13T01:46:27.573273Z","shell.execute_reply.started":"2025-06-13T01:46:27.366216Z","shell.execute_reply":"2025-06-13T01:46:27.572281Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7. Thermopile & ToF Visualization\n# 7‑1. Thermopile Temperature Hist","metadata":{}},{"cell_type":"code","source":"thm_cols = [f\"thm_{i}\" for i in range(1,6)]\ntrain[thm_cols].hist(bins=50, figsize=(12,4), layout=(1,5))\nplt.suptitle(\"Thermopile temperature distribution (°C)\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:27.574345Z","iopub.execute_input":"2025-06-13T01:46:27.574659Z","iopub.status.idle":"2025-06-13T01:46:28.828459Z","shell.execute_reply.started":"2025-06-13T01:46:27.574637Z","shell.execute_reply":"2025-06-13T01:46:28.827548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 7‑2. ToF 8×8 heatmap example","metadata":{}},{"cell_type":"code","source":"# --- 1. Required libraries & inline magic ---\n%matplotlib inline\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# --- 2. Data loading & creating seq_level ---\nDATA_DIR = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\ntrain    = pd.read_csv(f\"{DATA_DIR}/train.csv\")\n\n# count the number of rows for each sequence_id\nseq_level = train.groupby(\"sequence_id\")[\"row_id\"].count().rename(\"rows_in_seq\")\n\n# --- 3. Automatically select sequences that can be visualized ---\n# Here we look for items with more than 100 lines\ncandidate = seq_level[seq_level >= 100]\nif len(candidate)==0:\n    raise ValueError(\"No sequences found with more than 100 rows\")\nEXAMPLE_SEQ = candidate.index[0]\nprint(f\"Selected sequence {EXAMPLE_SEQ} has number of lines {candidate.iloc[0]} \")\n\n# --- 4. Frame extraction & index clip ---\nseq_df    = train[train[\"sequence_id\"] == EXAMPLE_SEQ].reset_index(drop=True)\ndesired_idx = 100\nframe_idx  = min(desired_idx, len(seq_df)-1)  # Prevent out-of-bounds\nprint(f\"Frame index to actually plot:{frame_idx}\")\n\nrow = seq_df.iloc[frame_idx]\n\n# --- 5. Draw a heat map of 5 ToF ---\nfor sensor_id in range(1, 6):\n    cols   = [f\"tof_{sensor_id}_v{i}\" for i in range(64)]\n    pixels = row[cols].values.astype(float)\n    pixels[pixels == -1] = np.nan\n    grid   = pixels.reshape(8, 8)\n\n    plt.figure(figsize=(3, 3))\n    sns.heatmap(\n        grid,\n        vmin=0, vmax=254,\n        cmap=\"viridis\",\n        cbar=False,\n        square=True,\n        linewidth=0.1,\n        linecolor=\"gray\",\n    )\n    plt.title(f\"Seq {EXAMPLE_SEQ}  Frame {frame_idx}  ToF {sensor_id}\", fontsize=10)\n    plt.axis(\"off\")\n    plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-22T13:49:08.566585Z","iopub.execute_input":"2025-07-22T13:49:08.566980Z","iopub.status.idle":"2025-07-22T13:50:02.931939Z","shell.execute_reply.started":"2025-07-22T13:49:08.566945Z","shell.execute_reply":"2025-07-22T13:50:02.930892Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Combining with subject attributes\nInsight example\n\nUnderstand sample balance by children vs. adults, men vs. women\n\nThis will serve as a stepping stone to later verify the relationship between height/arm length and gesture recognition accuracy.","metadata":{}},{"cell_type":"code","source":"# 1) Convert basic information for each sequence into a DataFrame\nseq_info = (\n    train\n    .groupby(\"sequence_id\")\n    .agg(\n        # Get \"gesture\", \"sequence_type\" and \"subject\" from the first line for each sequence_id\n        gesture        = (\"gesture\",       \"first\"),\n        sequence_type  = (\"sequence_type\", \"first\"),\n        rows_in_seq    = (\"row_id\",        \"count\"),\n        subject        = (\"subject\",       \"first\"),\n    )\n    .reset_index() # index=sequence_id → columnization\n)\n\n# 2) Load demographics (subject attributes) data\ntrain_demo = pd.read_csv(f\"{DATA_DIR}/train_demographics.csv\")\n\n# 3) Combine seq_info and train_demo by subject\nseq_demo = seq_info.merge(train_demo, on=\"subject\", how=\"left\")\n\n# 4) Count the number of sequences with adult_child (child/adult) and sex (gender)\npivot = seq_demo.groupby([\"adult_child\", \"sex\"])[\"sequence_id\"] \\\n                .count() \\\n                .unstack(fill_value=0)\n\nprint(pivot)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:50:18.644992Z","iopub.execute_input":"2025-06-13T01:50:18.645764Z","iopub.status.idle":"2025-06-13T01:50:18.884964Z","shell.execute_reply.started":"2025-06-13T01:50:18.645736Z","shell.execute_reply":"2025-06-13T01:50:18.883888Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 9. Identification of “IMU only vs. all sensors” section\n\nYou can consider strategies such as inserting a sensor dropout during model learning or splitting the data into two and training a separate model.","metadata":{}},{"cell_type":"code","source":"\n# If Thermopile is all NaN, it is determined as IMU-only\nis_imu_only = train.groupby(\"sequence_id\")[\"thm_1\"].apply(lambda x: x.isna().all())\nprint(is_imu_only.value_counts())\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-13T01:46:50.109289Z","iopub.status.idle":"2025-06-13T01:46:50.109557Z","shell.execute_reply.started":"2025-06-13T01:46:50.109418Z","shell.execute_reply":"2025-06-13T01:46:50.109430Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 10. Summary Checklist\n\n\n| Aspect             | Purpose                        | Code Example                                 |\n| -------------- | ------------------------- | ------------------------------------- |\n| **Data Volume & Memory** | Estimate training / inference time              | `train.memory_usage(deep=True).sum()` |\n| **Class Distribution**      | Gauge imbalance → weighting or sampling strategy | `value_counts(normalize=True)`        |\n| **Missingness Structur**       | Decide how to handle NULL or -1 (ToF)   | `missing.mean()`                      |\n| **Outliers**        | Set clipping thresholds                | `quantile([.001, .999])`              |\n| **Subject Leakage**     | Check for leakage → validate GroupKFold         | `seq_level['subject'].nunique()`      |\n| **Sensor Correlation**      | Compress highly correlated channels              | `train[imu_cols].corr()`              |\n","metadata":{}}]}