{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":246534658,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"d6df2627-9b55-41e9-b77d-198235edb2e2","cell_type":"markdown","source":"**Library - Packages**","metadata":{}},{"id":"de2c8c1b-6584-40d2-aadc-fdf320edd064","cell_type":"code","source":"# General use default library\nimport numpy as np\nimport pandas as pd\nimport time\n# visualisation library\nimport IPython.display\nfrom IPython.display import Image, display\nimport matplotlib\nfrom matplotlib import pyplot as plt\nimport itertools\nfrom itertools import cycle\nimport seaborn as sns\nfrom mpl_toolkits.mplot3d import Axes3D  # Needed for 3D plotting\nsns.set(rc={'figure.figsize':(12,14)})\nsns.set_theme()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:29.521666Z","iopub.execute_input":"2025-06-20T19:31:29.522010Z","iopub.status.idle":"2025-06-20T19:31:30.969612Z","shell.execute_reply.started":"2025-06-20T19:31:29.521986Z","shell.execute_reply":"2025-06-20T19:31:30.968704Z"}},"outputs":[],"execution_count":null},{"id":"b12bd80b-3aef-4200-96eb-65bc8e06dbfc","cell_type":"markdown","source":"**Load and annex notebook**","metadata":{}},{"id":"a13dff33-bebf-4e4a-a21b-f7678db15dc5","cell_type":"code","source":"import cmi_comp_utils_functions as my_utils","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:30.972198Z","iopub.execute_input":"2025-06-20T19:31:30.972751Z","iopub.status.idle":"2025-06-20T19:31:44.207349Z","shell.execute_reply.started":"2025-06-20T19:31:30.972729Z","shell.execute_reply":"2025-06-20T19:31:44.206446Z"}},"outputs":[],"execution_count":null},{"id":"abc27d72-6cf8-44fc-816d-fd35fe5c72a7","cell_type":"markdown","source":"# **Let's inspect the demographics train/test datasets**","metadata":{}},{"id":"ecf4b95b-c297-47cf-bfb8-5b2e4df6ea16","cell_type":"markdown","source":"## **Train dataset**","metadata":{}},{"id":"9b9411df-fc07-4bfb-9f4f-4a8197257b61","cell_type":"code","source":"filepath = my_utils.Path('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\nraw_train_demographics_df = pd.read_csv(filepath, sep=',', encoding='utf-8', na_filter=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.208369Z","iopub.execute_input":"2025-06-20T19:31:44.208881Z","iopub.status.idle":"2025-06-20T19:31:44.223088Z","shell.execute_reply.started":"2025-06-20T19:31:44.208856Z","shell.execute_reply":"2025-06-20T19:31:44.222173Z"}},"outputs":[],"execution_count":null},{"id":"68464378-3665-4e4c-bc60-c2369bc2311a","cell_type":"code","source":"raw_train_demographics_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.224137Z","iopub.execute_input":"2025-06-20T19:31:44.224464Z","iopub.status.idle":"2025-06-20T19:31:44.232370Z","shell.execute_reply.started":"2025-06-20T19:31:44.224442Z","shell.execute_reply":"2025-06-20T19:31:44.231537Z"}},"outputs":[],"execution_count":null},{"id":"37cafd5c-e7f2-4a3e-a2e3-cb04adc82e77","cell_type":"code","source":"raw_train_demographics_df.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.234690Z","iopub.execute_input":"2025-06-20T19:31:44.235064Z","iopub.status.idle":"2025-06-20T19:31:44.272039Z","shell.execute_reply.started":"2025-06-20T19:31:44.235041Z","shell.execute_reply":"2025-06-20T19:31:44.270938Z"}},"outputs":[],"execution_count":null},{"id":"8af2bae6","cell_type":"code","source":"train_shape = raw_train_demographics_df.shape\ntrain_dtypes = raw_train_demographics_df.dtypes\nmissing_counts = raw_train_demographics_df.isna().sum()\nmissing_percent = ((raw_train_demographics_df.isnull().sum() / len(raw_train_demographics_df)) * 100).sort_values()\n\nsummary = f\"\"\"\nTrain Demographics Dataset Summary:\n- Shape: {train_shape[0]} rows, {train_shape[1]} columns\n- Data types:\n{train_dtypes.to_string()}\n- Missing values (count):\n{missing_counts.to_string()}\n- Missing values (percentage):\n{missing_percent.to_string()}\n\n\"\"\"\nprint(summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.273102Z","iopub.execute_input":"2025-06-20T19:31:44.273397Z","iopub.status.idle":"2025-06-20T19:31:44.283835Z","shell.execute_reply.started":"2025-06-20T19:31:44.273367Z","shell.execute_reply":"2025-06-20T19:31:44.283025Z"}},"outputs":[],"execution_count":null},{"id":"fd9fc16f-eacf-407f-9b6e-10225b9a9fe7","cell_type":"code","source":"TRAIN_DEMO_COLUMNS = [\n    'subject',\n    'adult_child',       \n    'age',    \n    'sex',     \n    'handedness',     \n    'height_cm',        \n    'shoulder_to_wrist_cm', \n    'elbow_to_wrist_cm', \n]\nprint(TRAIN_DEMO_COLUMNS)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.284877Z","iopub.execute_input":"2025-06-20T19:31:44.285282Z","iopub.status.idle":"2025-06-20T19:31:44.305598Z","shell.execute_reply.started":"2025-06-20T19:31:44.285260Z","shell.execute_reply":"2025-06-20T19:31:44.304593Z"}},"outputs":[],"execution_count":null},{"id":"debabf66-a1be-429e-a44a-7b11dcd5a8e3","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>Overall, the dataset has no missing values in any column, as all percentages are 0%.</b>\n</div>","metadata":{}},{"id":"393ef26c-9e22-4c15-9708-f16644e1e6ee","cell_type":"code","source":"sns.pairplot(raw_train_demographics_df[TRAIN_DEMO_COLUMNS], corner=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:44.306620Z","iopub.execute_input":"2025-06-20T19:31:44.306916Z","iopub.status.idle":"2025-06-20T19:31:52.458979Z","shell.execute_reply.started":"2025-06-20T19:31:44.306890Z","shell.execute_reply":"2025-06-20T19:31:52.457778Z"}},"outputs":[],"execution_count":null},{"id":"fcef242c-7b12-40de-928a-4dde320d2bcc","cell_type":"markdown","source":"- **Sex, Adult child and handedness**","metadata":{}},{"id":"7025afa7-0cad-4fa5-8c6c-4f9206d2fee2","cell_type":"code","source":"# Plot 3 histograms\nmy_utils.display_3x_histoplot(raw_train_demographics_df, ['sex', 'adult_child', 'handedness'], figsize=(18, 5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:52.460188Z","iopub.execute_input":"2025-06-20T19:31:52.460459Z","iopub.status.idle":"2025-06-20T19:31:53.374627Z","shell.execute_reply.started":"2025-06-20T19:31:52.460431Z","shell.execute_reply":"2025-06-20T19:31:53.373668Z"}},"outputs":[],"execution_count":null},{"id":"cfd47400-ea02-497f-835b-57f75dc69b4f","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>Out of the 81 observations:</b>\n    <ul>\n        <li>\n            sex: Approximately 39% of the participants are women (sex = 0) and 61% are men (sex = 1).\n        </li>\n        <li>\n            Adult/Child: About 48% of the participants are children (adult_child = 0), and 52% are adults (adult_child = 1).\n        </li>\n        <li>\n            Handedness: Around 13% of participants are left-handed (handedness = 0), while 87% are right-handed (handedness = 1).\n        </li>\n    </ul>\n    <b>These statistics indicate a slight majority of male, adult, and right-handed participants in the dataset.</b>\n</div>","metadata":{}},{"id":"feb46724-a9c9-45bd-89b1-82a560c58333","cell_type":"markdown","source":"- **Age and Sex**","metadata":{}},{"id":"fa01c62c-39fb-4340-9ae0-1348acf7045f","cell_type":"code","source":"sns.histplot(data=raw_train_demographics_df, x=\"age\", hue=\"sex\", stat=\"percent\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:53.375494Z","iopub.execute_input":"2025-06-20T19:31:53.375723Z","iopub.status.idle":"2025-06-20T19:31:53.758381Z","shell.execute_reply.started":"2025-06-20T19:31:53.375705Z","shell.execute_reply":"2025-06-20T19:31:53.757347Z"}},"outputs":[],"execution_count":null},{"id":"b4fa8ae9-d358-47f9-9f5f-4beacd3d9006","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>The histogram shows the age distribution of participants in the training dataset, separated by sex (0 = female, 1 = male). The y-axis represents the percentage of participants in each age bin.</b>\n    <b>Key observations:</b>\n    <ul>\n        <li>\n            Most participants under age 18 are female (sex = 0).\n        </li>\n        <li>\n            Males (sex = 1) are more prevalent in the 20–35 age range and are the only group present from ages 40 to 45.\n        </li>\n        <li>\n            Around age 50, the number of male and female participants is equal.\n        </li>\n        <li>\n            The distribution is right-skewed, with more participants in the younger age groups, especially among females.\n        </li>\n    </ul>\n</div>","metadata":{}},{"id":"2b3bc4db-dae2-478f-bd4d-96d4adef689d","cell_type":"markdown","source":"- **Height and Sex**","metadata":{}},{"id":"307a7485-bd98-45db-9e5c-c85eae791883","cell_type":"code","source":"sns.histplot(data=raw_train_demographics_df, x=\"height_cm\", hue=\"sex\", stat=\"percent\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:53.759385Z","iopub.execute_input":"2025-06-20T19:31:53.759772Z","iopub.status.idle":"2025-06-20T19:31:54.129300Z","shell.execute_reply.started":"2025-06-20T19:31:53.759744Z","shell.execute_reply":"2025-06-20T19:31:54.128190Z"}},"outputs":[],"execution_count":null},{"id":"62185650","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>The histogram shows the height distribution of participants in the training dataset, separated by sex (0 = female, 1 = male). The y-axis represents the percentage of participants in each height bin.</b>\n    <b>Key observations:</b>\n    <ul>\n        <li>\n            Female participants (sex = 0) are predominantly found in the 150–170 cm height range, with very few above 175 cm.\n        </li>\n        <li>\n            Male participants (sex = 1) are most common in the 170–180 cm height range, showing a clear peak around 170 cm.\n        </li>\n        <li>\n            Heights above 175 cm are almost exclusively male, while heights below 155 cm are more frequently observed among females.\n        </li>\n        <li>\n            There is a visible overlap between sexes in the 160–175 cm range, but males tend to be taller overall.\n        </li>\n    </ul>\n</div>","metadata":{"vscode":{"languageId":"markdown"}}},{"id":"45bac612-5705-40f6-a7f6-763d7319aeca","cell_type":"markdown","source":"- **shoulder_to_wrist_cm and elbow_to_wrist_cm**","metadata":{}},{"id":"30153b7a","cell_type":"code","source":"# Scatter plot: shoulder_to_wrist_cm vs elbow_to_wrist_cm\nfig, axes = plt.subplots(1, 1, figsize=(6, 6))\nsns.scatterplot(\n    data=raw_train_demographics_df,\n    x=\"shoulder_to_wrist_cm\",\n    y=\"elbow_to_wrist_cm\"\n)\nplt.title(\"Scatter Plot: Shoulder to Wrist vs Elbow to Wrist\")\nplt.xlabel(\"Shoulder to Wrist (cm)\")\nplt.ylabel(\"Elbow to Wrist (cm)\")\nplt.show()\n\n# Boxplot for both variables\nfig, axes = plt.subplots(1, 2, figsize=(12, 5))\n\nsns.boxplot(y=raw_train_demographics_df[\"shoulder_to_wrist_cm\"], ax=axes[0])\naxes[0].set_title(\"Boxplot: Shoulder to Wrist (cm)\")\nsns.boxplot(y=raw_train_demographics_df[\"elbow_to_wrist_cm\"], ax=axes[1])\naxes[1].set_title(\"Boxplot: Elbow to Wrist (cm)\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.130463Z","iopub.execute_input":"2025-06-20T19:31:54.130701Z","iopub.status.idle":"2025-06-20T19:31:54.686199Z","shell.execute_reply.started":"2025-06-20T19:31:54.130681Z","shell.execute_reply":"2025-06-20T19:31:54.685171Z"}},"outputs":[],"execution_count":null},{"id":"d776aa22","cell_type":"code","source":"# Calculate IQR for 'shoulder_to_wrist_cm'\n\nq1 = raw_train_demographics_df['shoulder_to_wrist_cm'].quantile(0.25)\nq3 = raw_train_demographics_df['shoulder_to_wrist_cm'].quantile(0.75)\niqr = q3 - q1\n\n# Define outlier bounds\nlower_bound = q1 - 1.5 * iqr\nupper_bound = q3 + 1.5 * iqr\n\n# Find outlier rows\noutliers = raw_train_demographics_df[\n    (raw_train_demographics_df['shoulder_to_wrist_cm'] < lower_bound) |\n    (raw_train_demographics_df['shoulder_to_wrist_cm'] > upper_bound)\n]\n\noutliers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.687352Z","iopub.execute_input":"2025-06-20T19:31:54.687684Z","iopub.status.idle":"2025-06-20T19:31:54.704242Z","shell.execute_reply.started":"2025-06-20T19:31:54.687657Z","shell.execute_reply":"2025-06-20T19:31:54.703217Z"}},"outputs":[],"execution_count":null},{"id":"eb977b0b","cell_type":"markdown","source":"**Boxplot Observations: Shoulder to Wrist (cm) and Elbow to Wrist (cm)**\n\n- **Shoulder to Wrist (cm):**\n    - The majority of values are concentrated between approximately 49 and 55 cm, as indicated by the interquartile range (IQR).\n    - There is one clear outlier above 70 cm, which stands out from the rest of the data.\n    - The distribution is slightly right-skewed, with the median closer to the lower quartile.\n    - Most data points fall within the whiskers, suggesting limited variability except for the outlier.\n","metadata":{}},{"id":"3686409f-c8b2-408b-aa68-3508e53e15a5","cell_type":"code","source":"# Calculate IQR for 'elbow_to_wrist_cm'\n\nq1 = raw_train_demographics_df['elbow_to_wrist_cm'].quantile(0.25)\nq3 = raw_train_demographics_df['elbow_to_wrist_cm'].quantile(0.75)\niqr = q3 - q1\n\n# Define outlier bounds\nlower_bound = q1 - 1.5 * iqr\nupper_bound = q3 + 1.5 * iqr\n\n# Find outlier rows\noutliers = raw_train_demographics_df[\n    (raw_train_demographics_df['elbow_to_wrist_cm'] < lower_bound) |\n    (raw_train_demographics_df['elbow_to_wrist_cm'] > upper_bound)\n]\n\noutliers","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.707382Z","iopub.execute_input":"2025-06-20T19:31:54.707648Z","iopub.status.idle":"2025-06-20T19:31:54.732566Z","shell.execute_reply.started":"2025-06-20T19:31:54.707629Z","shell.execute_reply":"2025-06-20T19:31:54.731619Z"}},"outputs":[],"execution_count":null},{"id":"7dbe03f6-ad44-4d3e-a6a9-582a1ad58320","cell_type":"markdown","source":"- **Elbow to Wrist (cm):**\n    - Most values are distributed between roughly 24 and 27 cm.\n    - There are two outliers: one below 20 cm and another above 40 cm.\n    - The median is near the center of the box, indicating a more symmetric distribution for the central data.\n    - The spread of the data is smaller compared to the shoulder-to-wrist measurements, but the outliers indicate a few unusual cases.","metadata":{}},{"id":"60bb99f9-c09a-4f90-859c-dcf15c907f83","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>Summary:</b>\n    <ul>\n     Both variables show a generally consistent range for most participants, with a few outliers present. The shoulder-to-wrist measurement has a single high outlier, while the elbow-to-wrist measurement has both a low and a high outlier. These outliers may warrant further investigation to determine if they are data entry errors or true anatomical variations.\n    </ul>\n</div>","metadata":{}},{"id":"f91e3f4a-d577-43d2-83d8-b892a721694d","cell_type":"markdown","source":"## **Test dataset**","metadata":{}},{"id":"96167d65-b5ae-4f34-a1b1-b73e94859467","cell_type":"code","source":"filepath = my_utils.Path('/kaggle/usr/lib/cmi_comp_utils_functions/cmi-competition-data/raw-data/test_demographics.csv')\nraw_test_demographics_df = pd.read_csv(filepath, sep=',', encoding='utf-8', na_filter=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.733834Z","iopub.execute_input":"2025-06-20T19:31:54.734821Z","iopub.status.idle":"2025-06-20T19:31:54.750111Z","shell.execute_reply.started":"2025-06-20T19:31:54.734780Z","shell.execute_reply":"2025-06-20T19:31:54.749299Z"}},"outputs":[],"execution_count":null},{"id":"8bbb62d4-6ad8-4dfd-9569-503464a38c5c","cell_type":"code","source":"raw_test_demographics_df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.751280Z","iopub.execute_input":"2025-06-20T19:31:54.751614Z","iopub.status.idle":"2025-06-20T19:31:54.757982Z","shell.execute_reply.started":"2025-06-20T19:31:54.751584Z","shell.execute_reply":"2025-06-20T19:31:54.757206Z"}},"outputs":[],"execution_count":null},{"id":"14f45087-1a21-40a4-86d1-e855b15a5c85","cell_type":"code","source":"raw_test_demographics_df.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-20T19:31:54.758999Z","iopub.execute_input":"2025-06-20T19:31:54.759883Z","iopub.status.idle":"2025-06-20T19:31:54.782736Z","shell.execute_reply.started":"2025-06-20T19:31:54.759860Z","shell.execute_reply":"2025-06-20T19:31:54.781745Z"}},"outputs":[],"execution_count":null},{"id":"814a4ad8-3ac2-4e0d-8f08-efc7a0c57549","cell_type":"markdown","source":"<div class=\"alert alert-success\">\n    <b>Out of the 2 observations:</b>\n    <ul>\n        <li>\n            The male test subject is a right handed adult .\n        </li>\n        <li>\n            Male participants (sex = 1) dominate the 170–180 cm height range, with a noticeable peak around 170 cm.\n        </li>\n        <li>\n            Heights above 175 cm are primarily occupied by males, while shorter heights below 155 cm are more frequently observed in females.\n        </li>\n    </ul>\n</div>","metadata":{}}]}