{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nimport os\nfrom scipy import stats\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:24:41.473764Z","iopub.execute_input":"2025-07-08T05:24:41.474137Z","iopub.status.idle":"2025-07-08T05:24:42.136195Z","shell.execute_reply.started":"2025-07-08T05:24:41.474105Z","shell.execute_reply":"2025-07-08T05:24:42.135368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\ntest_data = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\ntest_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T04:38:42.197464Z","iopub.execute_input":"2025-07-08T04:38:42.198277Z","iopub.status.idle":"2025-07-08T04:39:10.823106Z","shell.execute_reply.started":"2025-07-08T04:38:42.198233Z","shell.execute_reply":"2025-07-08T04:39:10.821961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T04:48:01.303947Z","iopub.execute_input":"2025-07-08T04:48:01.304266Z","iopub.status.idle":"2025-07-08T04:48:01.331280Z","shell.execute_reply.started":"2025-07-08T04:48:01.304242Z","shell.execute_reply":"2025-07-08T04:48:01.330334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_demo.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T04:41:56.264397Z","iopub.execute_input":"2025-07-08T04:41:56.264767Z","iopub.status.idle":"2025-07-08T04:41:56.278130Z","shell.execute_reply.started":"2025-07-08T04:41:56.264739Z","shell.execute_reply":"2025-07-08T04:41:56.277181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_data.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T04:42:16.219944Z","iopub.execute_input":"2025-07-08T04:42:16.220238Z","iopub.status.idle":"2025-07-08T04:42:16.244187Z","shell.execute_reply.started":"2025-07-08T04:42:16.220217Z","shell.execute_reply":"2025-07-08T04:42:16.243446Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_demo.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T04:42:26.167621Z","iopub.execute_input":"2025-07-08T04:42:26.167973Z","iopub.status.idle":"2025-07-08T04:42:26.179086Z","shell.execute_reply.started":"2025-07-08T04:42:26.167949Z","shell.execute_reply":"2025-07-08T04:42:26.177827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def explore_basic_info(df, name):\n    \"\"\"Explore basic information of the dataset\"\"\"\n    print(f\"\\n===== Basic Information of {name} =====\")\n    print(f\"Data shape: {df.shape}\")\n    print(f\"Column names: {df.columns.tolist()[:10]}...\")\n    print(\"\\nData types:\")\n    print(df.dtypes.value_counts())\n    print(\"\\nMissing value statistics:\")\n    missing = df.isnull().sum()\n    missing = missing[missing > 0]\n    if len(missing) > 0:\n        print(missing)\n    else:\n        print(\"No missing values\")\n    \n    # Check for negative values (negative values may indicate anomalies in sensor data)\n    numeric_cols = df.select_dtypes(include=[np.number]).columns\n    neg_counts = (df[numeric_cols] < 0).sum()\n    neg_counts = neg_counts[neg_counts > 0]\n    if len(neg_counts) > 0:\n        print(\"\\nColumns containing negative values:\")\n        print(neg_counts)\n\nexplore_basic_info(train_data, \"Train data\")\nexplore_basic_info(test_data, \"Test data\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:01:44.439436Z","iopub.execute_input":"2025-07-08T05:01:44.439746Z","iopub.status.idle":"2025-07-08T05:01:46.670044Z","shell.execute_reply.started":"2025-07-08T05:01:44.439723Z","shell.execute_reply":"2025-07-08T05:01:46.669050Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"adult_child_counts = train_demo['adult_child'].value_counts()\nprint(\"Adult vs. Child Ratio:\")\nprint(f\"Adult (1): {adult_child_counts.get(1, 0)}\")\nprint(f\"Child (0): {adult_child_counts.get(0, 0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:42:04.798769Z","iopub.execute_input":"2025-07-08T05:42:04.799131Z","iopub.status.idle":"2025-07-08T05:42:04.805956Z","shell.execute_reply.started":"2025-07-08T05:42:04.799105Z","shell.execute_reply":"2025-07-08T05:42:04.805025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gender distribution\nsex_counts = train_demo['sex'].value_counts()\nprint(\"\\nGender Distribution:\")\nprint(f\"Male (1): {sex_counts.get(1, 0)}\")\nprint(f\"Female (0): {sex_counts.get(0, 0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:43:00.488745Z","iopub.execute_input":"2025-07-08T05:43:00.489044Z","iopub.status.idle":"2025-07-08T05:43:00.495995Z","shell.execute_reply.started":"2025-07-08T05:43:00.489025Z","shell.execute_reply":"2025-07-08T05:43:00.494830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Handedness distribution\nhand_counts = train_demo['handedness'].value_counts()\nprint(\"\\nHandedness Distribution:\")\nprint(f\"Right-handed (1): {hand_counts.get(1, 0)}\")\nprint(f\"Left-handed (0): {hand_counts.get(0, 0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:43:21.294059Z","iopub.execute_input":"2025-07-08T05:43:21.294386Z","iopub.status.idle":"2025-07-08T05:43:21.301849Z","shell.execute_reply.started":"2025-07-08T05:43:21.294361Z","shell.execute_reply":"2025-07-08T05:43:21.300890Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Age distribution\nprint(\"\\nAge statistics:\")\nprint(train_demo['age'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:43:42.757404Z","iopub.execute_input":"2025-07-08T05:43:42.757795Z","iopub.status.idle":"2025-07-08T05:43:42.766322Z","shell.execute_reply.started":"2025-07-08T05:43:42.757769Z","shell.execute_reply":"2025-07-08T05:43:42.765483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Height distribution\nprint(\"\\nHeight statistics (cm):\")\nprint(train_demo['height_cm'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:44:11.228549Z","iopub.execute_input":"2025-07-08T05:44:11.228905Z","iopub.status.idle":"2025-07-08T05:44:11.237552Z","shell.execute_reply.started":"2025-07-08T05:44:11.228879Z","shell.execute_reply":"2025-07-08T05:44:11.236606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualization\nplt.figure(figsize=(15, 12))\n\n# Age distribution histogram\nplt.subplot(2, 2, 1)\nsns.histplot(train_demo['age'], kde=True)\nplt.title('Age Distribution')\nplt.xlabel('Age')\nplt.ylabel('Frequency')\n\n# Height distribution histogram\nplt.subplot(2, 2, 2)\nsns.histplot(train_demo['height_cm'], kde=True)\nplt.title('Height Distribution (cm)')\nplt.xlabel('Height (cm)')\nplt.ylabel('Frequency')\n\n\n# Adult/child pie chart\nplt.subplot(2, 2, 3)\nplt.pie(adult_child_counts, labels=['Child', 'Adult'] if 0 in adult_child_counts else ['Adult', 'Child'], \n        autopct='%1.1f%%', startangle=90)\nplt.title('Adult vs. Child Ratio')\n\n# Gender pie chart\nplt.subplot(2, 2, 4)\nplt.pie(sex_counts, labels=['Female', 'Male'] if 0 in sex_counts else ['Male', 'Female'], \n        autopct='%1.1f%%', startangle=90)\nplt.title('Gender Distribution')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:45:02.780454Z","iopub.execute_input":"2025-07-08T05:45:02.781147Z","iopub.status.idle":"2025-07-08T05:45:03.458535Z","shell.execute_reply.started":"2025-07-08T05:45:02.781121Z","shell.execute_reply":"2025-07-08T05:45:03.457238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract columns for different types of sensor data\nacc_cols = [col for col in train_data.columns if col.startswith('acc_')]\nrot_cols = [col for col in train_data.columns if col.startswith('rot_')]\nthm_cols = [col for col in train_data.columns if col.startswith('thm_')]\ntof_cols = [col for col in train_data.columns if col.startswith('tof_')]\n\nprint(f\"Number of accelerometer features: {len(acc_cols)}\")\nprint(f\"Number of rotation features: {len(rot_cols)}\")\nprint(f\"Number of thermometer sensor features: {len(thm_cols)}\")\nprint(f\"Number of TOF sensor features: {len(tof_cols)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:45:30.492791Z","iopub.execute_input":"2025-07-08T05:45:30.493123Z","iopub.status.idle":"2025-07-08T05:45:30.499998Z","shell.execute_reply.started":"2025-07-08T05:45:30.493100Z","shell.execute_reply":"2025-07-08T05:45:30.499071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Analyze accelerometer data\nprint(\"\\nAccelerometer Data Statistics:\")\nacc_data = train_data[acc_cols].describe().T\nprint(acc_data[['mean', 'std', 'min', 'max']])\n\n# Analyze rotation data\nprint(\"\\nRotation Data Statistics:\")\nrot_data = train_data[rot_cols].describe().T\nprint(rot_data[['mean', 'std', 'min', 'max']])\n\n# Analyze thermometer sensor data\nprint(\"\\nThermometer Sensor Data Statistics:\")\nthm_data = train_data[thm_cols].describe().T\nprint(thm_data[['mean', 'std', 'min', 'max']])\n\n# Analyze TOF sensor data \nprint(\"\\nTOF Sensor Data Statistics (First 10):\")\ntof_sample = tof_cols[:10]\ntof_data = train_data[tof_sample].describe().T\nprint(tof_data[['mean', 'std', 'min', 'max']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:47:48.079091Z","iopub.execute_input":"2025-07-08T05:47:48.079452Z","iopub.status.idle":"2025-07-08T05:47:48.677863Z","shell.execute_reply.started":"2025-07-08T05:47:48.079426Z","shell.execute_reply":"2025-07-08T05:47:48.677004Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize sensor data distributions\nplt.figure(figsize=(16, 12))\n\n# Accelerometer data distribution\nplt.subplot(2, 2, 1)\ntrain_data[acc_cols].boxplot()\nplt.title('Accelerometer Data Distribution')\nplt.xticks(rotation=90)\n\n# Rotation data distribution\nplt.subplot(2, 2, 2)\ntrain_data[rot_cols].boxplot()\nplt.title('Rotation Data Distribution')\nplt.xticks(rotation=90)\n\n# Thermometer sensor data distribution\nplt.subplot(2, 2, 3)\ntrain_data[thm_cols].boxplot()\nplt.title('Thermometer Sensor Data Distribution')\nplt.xticks(rotation=90)\n\n# TOF sensor data distribution \nplt.subplot(2, 2, 4)\ntrain_data[tof_sample].boxplot()\nplt.title('TOF Sensor Data Distribution (Sampled)')\nplt.xticks(rotation=90)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:48:21.628785Z","iopub.execute_input":"2025-07-08T05:48:21.629098Z","iopub.status.idle":"2025-07-08T05:48:33.476013Z","shell.execute_reply.started":"2025-07-08T05:48:21.629075Z","shell.execute_reply":"2025-07-08T05:48:33.475059Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Behavior distribution\nbehavior_counts = train_data['behavior'].value_counts()\nprint(\"Behavior Distribution:\")\nprint(behavior_counts)\n\n# Gesture distribution\ngesture_counts = train_data['gesture'].value_counts()\nprint(\"\\nGesture Distribution:\")\nprint(gesture_counts)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:48:54.192374Z","iopub.execute_input":"2025-07-08T05:48:54.192733Z","iopub.status.idle":"2025-07-08T05:48:54.296173Z","shell.execute_reply.started":"2025-07-08T05:48:54.192707Z","shell.execute_reply":"2025-07-08T05:48:54.295231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualization\nplt.figure(figsize=(15, 10))\n\n# Behavior distribution bar chart\nplt.subplot(2, 1, 1)\nbehavior_counts.plot(kind='bar')\nplt.title('Behavior Distribution')\nplt.xlabel('Behavior Type')\nplt.ylabel('Frequency')\nplt.xticks(rotation=45)\n\n# Gesture distribution bar chart\nplt.subplot(2, 1, 2)\ngesture_counts.plot(kind='bar')\nplt.title('Gesture Distribution')\nplt.xlabel('Gesture Type')\nplt.ylabel('Frequency')\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:49:20.091028Z","iopub.execute_input":"2025-07-08T05:49:20.091692Z","iopub.status.idle":"2025-07-08T05:49:20.623134Z","shell.execute_reply.started":"2025-07-08T05:49:20.091667Z","shell.execute_reply":"2025-07-08T05:49:20.622029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate the length of each sequence\nsequence_lengths = train_data.groupby('sequence_id').size()\n\nprint(f\"Total number of sequences: {len(sequence_lengths)}\")\nprint(f\"Shortest sequence length: {sequence_lengths.min()}\")\nprint(f\"Longest sequence length: {sequence_lengths.max()}\")\nprint(f\"Average sequence length: {sequence_lengths.mean():.2f}\")\nprint(f\"Median sequence length: {sequence_lengths.median()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:49:49.141909Z","iopub.execute_input":"2025-07-08T05:49:49.142233Z","iopub.status.idle":"2025-07-08T05:49:49.189748Z","shell.execute_reply.started":"2025-07-08T05:49:49.142208Z","shell.execute_reply":"2025-07-08T05:49:49.188472Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize sequence length distribution\nplt.figure(figsize=(10, 6))\nsns.histplot(sequence_lengths, kde=True)\nplt.title('Sequence Length Distribution')\nplt.xlabel('Sequence Length')\nplt.ylabel('Frequency')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:50:24.590981Z","iopub.execute_input":"2025-07-08T05:50:24.591263Z","iopub.status.idle":"2025-07-08T05:50:25.157641Z","shell.execute_reply.started":"2025-07-08T05:50:24.591242Z","shell.execute_reply":"2025-07-08T05:50:25.156704Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Analyze the number of sequences per subject\nif 'subject' in train_data.columns:\n    subject_sequences = train_data.groupby('subject')['sequence_id'].nunique()\n    \n    print(f\"\\nStatistics on the number of sequences per subject:\")\n    print(f\"Minimum number of sequences: {subject_sequences.min()}\")\n    print(f\"Maximum number of sequences: {subject_sequences.max()}\")\n    print(f\"Average number of sequences: {subject_sequences.mean():.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:50:49.698277Z","iopub.execute_input":"2025-07-08T05:50:49.698630Z","iopub.status.idle":"2025-07-08T05:50:49.779194Z","shell.execute_reply.started":"2025-07-08T05:50:49.698606Z","shell.execute_reply":"2025-07-08T05:50:49.778333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select numeric columns\nnumeric_cols = train_data.select_dtypes(include=[np.number]).columns.tolist()\n\n# Exclude ID and counter columns from numeric columns\nexclude_cols = ['row_id', 'sequence_counter']\nfeature_cols = [col for col in numeric_cols if col not in exclude_cols]\n\n# If there are too many features, select a subset for correlation analysis\nif len(feature_cols) > 20:\n    print(\"Too many features. Selecting a subset of important features for correlation analysis.\")\n    # Select acceleration, rotation, and temperature features\n    selected_features = [col for col in feature_cols if any(col.startswith(prefix) for prefix in ['acc_', 'rot_', 'thm_'])]\n    # If still too many, take the first 20\n    if len(selected_features) > 20:\n        selected_features = selected_features[:20]\nelse:\n    selected_features = feature_cols\n\n# Calculate correlation matrix\ncorrelation = train_data[selected_features].corr()\n\n# Visualize correlation matrix\nplt.figure(figsize=(12, 10))\nsns.heatmap(correlation, annot=False, cmap='coolwarm', vmin=-1, vmax=1)\nplt.title('Feature Correlation Heatmap')\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:51:11.419319Z","iopub.execute_input":"2025-07-08T05:51:11.419873Z","iopub.status.idle":"2025-07-08T05:51:12.881581Z","shell.execute_reply.started":"2025-07-08T05:51:11.419847Z","shell.execute_reply":"2025-07-08T05:51:12.880543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select numeric features\nnumeric_cols = train_data.select_dtypes(include=[np.number]).columns.tolist()\n\n# Exclude ID and counter columns\nexclude_cols = ['row_id', 'sequence_counter']\nfeature_cols = [col for col in numeric_cols if col not in exclude_cols]\n\n# Exclude target variables if present\nif 'behavior' in train_data.columns and 'behavior' in feature_cols:\n    feature_cols.remove('behavior')\nif 'gesture' in train_data.columns and 'gesture' in feature_cols:\n    feature_cols.remove('gesture')\n\nprint(f\"Number of features for PCA: {len(feature_cols)}\")\n\n# Extract features and handle missing values\nX = train_data[feature_cols].fillna(0)  # Fill missing values with 0 for simplicity\n\n# Standardize the data\nscaler = StandardScaler()\nX_scaled = scaler.fit_transform(X)\n\n# Apply PCA\npca = PCA()\nX_pca = pca.fit_transform(X_scaled)\n\n# Calculate explained variance ratio\nexplained_variance = pca.explained_variance_ratio_\ncumulative_variance = np.cumsum(explained_variance)\n\n# Find the number of components needed to explain 90% variance\nn_components_90 = np.argmax(cumulative_variance >= 0.9) + 1\n\nprint(f\"Number of principal components explaining 90% variance: {n_components_90}\")\nprint(f\"Variance explained by top 10 components: {explained_variance[:10]}\")\nprint(f\"Cumulative variance explained by top 10 components: {cumulative_variance[:10]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:51:53.396715Z","iopub.execute_input":"2025-07-08T05:51:53.397168Z","iopub.status.idle":"2025-07-08T05:52:22.923197Z","shell.execute_reply.started":"2025-07-08T05:51:53.397139Z","shell.execute_reply":"2025-07-08T05:52:22.922271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize explained variance\nplt.figure(figsize=(10, 6))\nplt.plot(range(1, len(explained_variance) + 1), cumulative_variance, marker='o', linestyle='-')\nplt.axhline(y=0.9, color='r', linestyle='--', label='90% Variance Threshold')\nplt.axvline(x=n_components_90, color='g', linestyle='--', label=f'{n_components_90} Components')\nplt.title('PCA Cumulative Explained Variance')\nplt.xlabel('Number of Principal Components')\nplt.ylabel('Cumulative Explained Variance Ratio')\nplt.grid(True)\nplt.legend()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:52:49.865287Z","iopub.execute_input":"2025-07-08T05:52:49.865586Z","iopub.status.idle":"2025-07-08T05:52:50.125279Z","shell.execute_reply.started":"2025-07-08T05:52:49.865562Z","shell.execute_reply":"2025-07-08T05:52:50.124411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Randomly select samples\nsample_size = min(5000, len(X_pca))  # Limit sample size to avoid performance issues\nindices = np.random.choice(len(X_pca), sample_size, replace=False)\n\n# Encode categorical variables to numerical values\nle = LabelEncoder()\ngesture_numeric = le.fit_transform(train_data['gesture'].iloc[indices])\n\n# Create scatter plot\nplt.figure(figsize=(12, 10))\nscatter = plt.scatter(X_pca[indices, 0], X_pca[indices, 1], \n                     c=gesture_numeric,\n                     cmap='tab20', alpha=0.6, s=30)\n\n# Add legend\nplt.legend(handles=scatter.legend_elements()[0], \n          labels=list(le.classes_),\n          title=\"Gesture Classes\",\n          bbox_to_anchor=(1.05, 1), loc='upper left')\n\nplt.title('Data Visualization after PCA Dimensionality Reduction')\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:53:33.323939Z","iopub.execute_input":"2025-07-08T05:53:33.324265Z","iopub.status.idle":"2025-07-08T05:53:33.908416Z","shell.execute_reply.started":"2025-07-08T05:53:33.324241Z","shell.execute_reply":"2025-07-08T05:53:33.907341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select numeric features\nnumeric_cols = train_data.select_dtypes(include=[np.number]).columns.tolist()\n\n# Exclude ID and counter columns\nexclude_cols = ['row_id', 'sequence_counter']\nfeature_cols = [col for col in numeric_cols if col not in exclude_cols]\n\n# If there are too many features, select a subset for outlier analysis\nif len(feature_cols) > 10:\n    # Select acceleration, rotation, and temperature features\n    selected_features = [col for col in feature_cols if any(col.startswith(prefix) for prefix in ['acc_', 'rot_', 'thm_'])]\n    # If still too many, take the first 10\n    if len(selected_features) > 10:\n        selected_features = selected_features[:10]\nelse:\n    selected_features = feature_cols\n\nprint(f\"Features for outlier analysis: {selected_features}\")\n\n# Detect outliers using Z-score method\nz_scores = stats.zscore(train_data[selected_features].fillna(0))\nabs_z_scores = np.abs(z_scores)\noutliers = (abs_z_scores > 3).any(axis=1)\noutlier_percentage = outliers.mean() * 100\n\nprint(f\"Percentage of outliers detected using Z-score method: {outlier_percentage:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:53:55.310270Z","iopub.execute_input":"2025-07-08T05:53:55.310568Z","iopub.status.idle":"2025-07-08T05:53:56.027382Z","shell.execute_reply.started":"2025-07-08T05:53:55.310545Z","shell.execute_reply":"2025-07-08T05:53:56.026578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualize box plots for each feature to show outliers\nplt.figure(figsize=(15, 10))\ntrain_data[selected_features].boxplot(vert=False)\nplt.title('Feature Box Plots - Outlier Detection')\nplt.xlabel('Value')\nplt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:54:43.584733Z","iopub.execute_input":"2025-07-08T05:54:43.585051Z","iopub.status.idle":"2025-07-08T05:54:49.212371Z","shell.execute_reply.started":"2025-07-08T05:54:43.585029Z","shell.execute_reply":"2025-07-08T05:54:49.211576Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Select several representative sequences for visualization\nsequence_ids = train_data['sequence_id'].unique()[:3]  # Select first 3 sequences\n\n# Select several sensor features for visualization\nacc_cols = [col for col in train_data.columns if col.startswith('acc_')][:3]  # First 3 acceleration features\nrot_cols = [col for col in train_data.columns if col.startswith('rot_')][:2]  # First 2 rotation features\nthm_cols = [col for col in train_data.columns if col.startswith('thm_')][:1]  # 1 temperature feature\n\nselected_features = acc_cols + rot_cols + thm_cols\n\n# Create time series plots for each selected sequence\nfor seq_id in sequence_ids:\n    seq_data = train_data[train_data['sequence_id'] == seq_id].sort_values('sequence_counter')\n    \n    plt.figure(figsize=(15, 10))\n    \n    for i, feature in enumerate(selected_features):\n        plt.subplot(len(selected_features), 1, i+1)\n        plt.plot(seq_data['sequence_counter'], seq_data[feature])\n        plt.title(f'{feature} - Sequence {seq_id}')\n        plt.xlabel('Sequence Counter')\n        plt.ylabel('Value')\n        plt.grid(True)\n    \n    plt.tight_layout()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-08T05:55:15.728333Z","iopub.execute_input":"2025-07-08T05:55:15.728622Z","iopub.status.idle":"2025-07-08T05:55:19.159092Z","shell.execute_reply.started":"2025-07-08T05:55:15.728599Z","shell.execute_reply":"2025-07-08T05:55:19.158100Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}