{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:24:32.008215Z","iopub.execute_input":"2025-07-17T15:24:32.008861Z","iopub.status.idle":"2025-07-17T15:24:32.022000Z","shell.execute_reply.started":"2025-07-17T15:24:32.008822Z","shell.execute_reply":"2025-07-17T15:24:32.020399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy import stats\nfrom scipy.signal import find_peaks, welch\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_selection import mutual_info_classif\nimport warnings\nwarnings.filterwarnings('ignore')\n\nplt.style.use('seaborn-v0_8')\nsns.set_palette(\"husl\")\n\nprint(\"Libraries loaded successfully\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:24:32.023835Z","iopub.execute_input":"2025-07-17T15:24:32.024186Z","iopub.status.idle":"2025-07-17T15:24:33.991956Z","shell.execute_reply.started":"2025-07-17T15:24:32.024153Z","shell.execute_reply":"2025-07-17T15:24:33.990607Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Data Loading and Basic Exploration","metadata":{}},{"cell_type":"code","source":"# Load sample of training data for analysis\nprint(\"Loading training data sample...\")\n\n# Load demographics first (smaller file)\ndemographics = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\nprint(f\"Demographics shape: {demographics.shape}\")\nprint(f\"Unique subjects: {demographics['subject'].nunique()}\")\n\n# Load a sample of main training data\nsample_size = 50000000  # Adjust based on memory\ntrain_sample = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv', nrows=sample_size)\nprint(f\"\\nTrain sample shape: {train_sample.shape}\")\nprint(f\"Unique sequences in sample: {train_sample['sequence_id'].nunique()}\")\nprint(f\"Unique gestures in sample: {train_sample['gesture'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:24:33.993560Z","iopub.execute_input":"2025-07-17T15:24:33.994101Z","iopub.status.idle":"2025-07-17T15:25:14.132629Z","shell.execute_reply.started":"2025-07-17T15:24:33.994072Z","shell.execute_reply":"2025-07-17T15:25:14.131499Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Basic data exploration\nprint(\"=== BASIC DATA OVERVIEW ===\")\nprint(\"\\nColumn types:\")\nprint(train_sample.dtypes.value_counts())\n\nprint(\"\\nMissing values:\")\nmissing_cols = train_sample.isnull().sum()\nmissing_cols = missing_cols[missing_cols > 0].sort_values(ascending=False)\nif len(missing_cols) > 0:\n    print(missing_cols.head(10))\nelse:\n    print(\"No missing values found\")\n\nprint(\"\\nSequence characteristics:\")\nseq_stats = train_sample.groupby('sequence_id').agg({\n    'sequence_counter': ['count', 'max'],\n    'gesture': 'first',\n    'sequence_type': 'first',\n    'subject': 'first'\n}).round(2)\n\nseq_stats.columns = ['length', 'max_counter', 'gesture', 'sequence_type', 'subject']\nprint(f\"Sequence length stats:\")\nprint(seq_stats['length'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:14.133760Z","iopub.execute_input":"2025-07-17T15:25:14.134332Z","iopub.status.idle":"2025-07-17T15:25:15.233604Z","shell.execute_reply.started":"2025-07-17T15:25:14.134300Z","shell.execute_reply":"2025-07-17T15:25:15.232821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Gesture distribution analysis\nprint(\"=== GESTURE ANALYSIS ===\")\n\ngesture_counts = train_sample['gesture'].value_counts()\nprint(\"\\nGesture distribution:\")\nfor gesture, count in gesture_counts.items():\n    print(f\"{gesture}: {count} ({count/len(train_sample)*100:.1f}%)\")\n\n# Sequence type distribution\nseq_type_counts = train_sample['sequence_type'].value_counts()\nprint(\"\\nSequence type distribution:\")\nprint(seq_type_counts)\n\n# Plot gesture distribution\nseq_gesture_counts = seq_stats['gesture'].value_counts()\ncolors1 = ['green'] * len(gesture_counts)   \ncolors2 = ['red'] * len(seq_gesture_counts)   \n\nplt.figure(figsize=(15, 8))\n\nplt.subplot(2, 1, 1)\ngesture_counts.plot(kind='bar', rot=45, color=colors1)\nplt.title('Gesture Distribution (by data points)')\nplt.ylabel('Count')\n\nplt.subplot(2, 1, 2)\nseq_gesture_counts.plot(kind='bar', rot=45, color=colors2)\nplt.title('Gesture Distribution (by sequences)')\nplt.ylabel('Number of Sequences')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:15.235877Z","iopub.execute_input":"2025-07-17T15:25:15.236179Z","iopub.status.idle":"2025-07-17T15:25:16.140061Z","shell.execute_reply.started":"2025-07-17T15:25:15.236157Z","shell.execute_reply":"2025-07-17T15:25:16.138921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Sensor Data Analysis","metadata":{}},{"cell_type":"code","source":"# IMU sensor analysis\nprint(\"=== IMU SENSOR ANALYSIS ===\")\n\nimu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\nprint(\"\\nIMU sensor statistics:\")\nimu_stats = train_sample[imu_cols].describe()\nprint(imu_stats)\n\n# Plot IMU distributions\nfig, axes = plt.subplots(2, 4, figsize=(20, 10))\naxes = axes.flatten()\n\ncolors = ['blue', 'salmon', 'green', 'gold', 'violet', 'orange', 'lightgreen']\n\nfor i, col in enumerate(imu_cols):\n    axes[i].hist(train_sample[col].dropna(), bins=50, alpha=0.7, color=\"green\")#colors[i])\n    axes[i].set_title(f'{col} Distribution')\n    axes[i].set_xlabel(col)\n    axes[i].set_ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n\n# Calculate acceleration magnitude\ntrain_sample['acc_magnitude'] = np.sqrt(\n    train_sample['acc_x']**2 + \n    train_sample['acc_y']**2 + \n    train_sample['acc_z']**2\n)\n\nprint(f\"\\nAcceleration magnitude stats:\")\nprint(train_sample['acc_magnitude'].describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:16.141272Z","iopub.execute_input":"2025-07-17T15:25:16.141625Z","iopub.status.idle":"2025-07-17T15:25:18.647279Z","shell.execute_reply.started":"2025-07-17T15:25:16.141576Z","shell.execute_reply":"2025-07-17T15:25:18.645954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Thermopile sensor analysis\nprint(\"=== THERMOPILE SENSOR ANALYSIS ===\")\n\nthm_cols = [f'thm_{i}' for i in range(1, 6)]\nprint(\"\\nThermopile sensor statistics:\")\nthm_stats = train_sample[thm_cols].describe()\nprint(thm_stats)\n\n# Plot thermopile distributions\nplt.figure(figsize=(15, 10))\n\nfor i, col in enumerate(thm_cols, 1):\n    plt.subplot(2, 3, i)\n    plt.hist(train_sample[col].dropna(), bins=50, alpha=0.7,color=\"orange\")#colors[i])\n    plt.title(f'{col} Distribution')\n    plt.xlabel('Temperature (°C)')\n    plt.ylabel('Frequency')\n\nplt.tight_layout()\nplt.show()\n\n# Thermopile correlation analysis\nplt.figure(figsize=(8, 6))\nthm_corr = train_sample[thm_cols].corr()\nsns.heatmap(thm_corr, annot=True, cmap='coolwarm', center=0)\nplt.title('Thermopile Sensor Correlations')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:18.649278Z","iopub.execute_input":"2025-07-17T15:25:18.649585Z","iopub.status.idle":"2025-07-17T15:25:20.707592Z","shell.execute_reply.started":"2025-07-17T15:25:18.649549Z","shell.execute_reply":"2025-07-17T15:25:20.706296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Time-of-Flight sensor analysis\nprint(\"=== TIME-OF-FLIGHT SENSOR ANALYSIS ===\")\n\n# Get ToF columns\ntof_cols = [col for col in train_sample.columns if col.startswith('tof_')]\nprint(f\"Number of ToF columns: {len(tof_cols)}\")\n\n# Analyze ToF data characteristics\ntof_sample = train_sample[tof_cols].sample(n=min(1000, len(train_sample)))\n\n# Count -1 values (no response)\nno_response_ratio = (tof_sample == -1).sum().sum() / (tof_sample.shape[0] * tof_sample.shape[1])\nprint(f\"\\nNo response ratio (-1 values): {no_response_ratio:.3f}\")\n\n# Valid ToF data statistics\ntof_valid = tof_sample.replace(-1, np.nan)\nprint(f\"\\nValid ToF data range: {tof_valid.min().min():.1f} - {tof_valid.max().max():.1f}\")\nprint(f\"Valid ToF data mean: {tof_valid.mean().mean():.1f}\")\n\n# Plot ToF sensor response distribution\nplt.figure(figsize=(12, 5))\n\nplt.subplot(1, 2, 1)\nvalid_values = tof_valid.values.flatten()\nvalid_values = valid_values[~np.isnan(valid_values)]\nplt.hist(valid_values, bins=50, alpha=0.7, color='Orange')\nplt.title('ToF Valid Values Distribution')\nplt.xlabel('Sensor Value')\nplt.ylabel('Frequency')\n\nplt.subplot(1, 2, 2)\nresponse_rates = (tof_sample != -1).mean(axis=0)\nplt.hist(response_rates, bins=30, alpha=0.7, color='green')\nplt.title('ToF Response Rate Distribution')\nplt.xlabel('Response Rate')\nplt.ylabel('Number of Pixels')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:20.708738Z","iopub.execute_input":"2025-07-17T15:25:20.709028Z","iopub.status.idle":"2025-07-17T15:25:21.950818Z","shell.execute_reply.started":"2025-07-17T15:25:20.709007Z","shell.execute_reply":"2025-07-17T15:25:21.949709Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Demographics Analysis","metadata":{}},{"cell_type":"code","source":"# Demographics analysis\nprint(\"=== DEMOGRAPHICS ANALYSIS ===\")\n\nprint(\"Demographics statistics:\")\nprint(demographics.describe())\n\nprint(\"\\nCategorical distributions:\")\nprint(f\"Adult/Child: {demographics['adult_child'].value_counts()}\")\nprint(f\"Sex: {demographics['sex'].value_counts()}\")\nprint(f\"Handedness: {demographics['handedness'].value_counts()}\")\n\n# Plot demographics\nfig, axes = plt.subplots(2, 3, figsize=(18, 12))\n\n# Age distribution\naxes[0, 0].hist(demographics['age'], bins=20, alpha=0.7, color= 'violet')\naxes[0, 0].set_title('Age Distribution')\naxes[0, 0].set_xlabel('Age (years)')\n\n# Height distribution\naxes[0, 1].hist(demographics['height_cm'], bins=20, alpha=0.7, color= 'violet')\naxes[0, 1].set_title('Height Distribution')\naxes[0, 1].set_xlabel('Height (cm)')\n\n# Arm measurements\naxes[0, 2].scatter(demographics['shoulder_to_wrist_cm'], demographics['elbow_to_wrist_cm'], alpha=0.6, color= 'red')\naxes[0, 2].set_title('Arm Measurements')\naxes[0, 2].set_xlabel('Shoulder to Wrist (cm)')\naxes[0, 2].set_ylabel('Elbow to Wrist (cm)')\n\n# Categorical distributions\ndemographics['adult_child'].value_counts().plot(kind='bar', ax=axes[1, 0], color= 'salmon')\naxes[1, 0].set_title('Adult vs Child')\naxes[1, 0].set_xticklabels(['1', '0'], rotation=0)\n\ndemographics['sex'].value_counts().plot(kind='bar', ax=axes[1, 1], color= 'salmon')\naxes[1, 1].set_title('Sex Distribution')\naxes[1, 1].set_xticklabels(['1', '0'], rotation=0)\n\ndemographics['handedness'].value_counts().plot(kind='bar', ax=axes[1, 2], color= 'salmon')\naxes[1, 2].set_title('Handedness Distribution')\naxes[1, 2].set_xticklabels(['1', '0'], rotation=0)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:26:57.614205Z","iopub.execute_input":"2025-07-17T15:26:57.614540Z","iopub.status.idle":"2025-07-17T15:26:58.772351Z","shell.execute_reply.started":"2025-07-17T15:26:57.614517Z","shell.execute_reply":"2025-07-17T15:26:58.771290Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Gesture-Specific Pattern Analysis","metadata":{}},{"cell_type":"code","source":"# Analyze patterns by gesture type\nprint(\"=== GESTURE-SPECIFIC ANALYSIS ===\")\n\n# Group by gesture and calculate statistics\ngesture_analysis = train_sample.groupby('gesture').agg({\n    'acc_magnitude': ['mean', 'std', 'max'],\n    'thm_1': ['mean', 'std'],\n    'thm_2': ['mean', 'std'],\n    'thm_3': ['mean', 'std'],\n    'sequence_counter': 'count'\n}).round(3)\n\ngesture_analysis.columns = ['_'.join(col).strip() for col in gesture_analysis.columns]\nprint(\"\\nGesture characteristics:\")\nprint(gesture_analysis)\n\n# Plot gesture-specific patterns\nfig, axes = plt.subplots(2, 2, figsize=(16, 12))\n\n# Acceleration magnitude by gesture\ngesture_acc = train_sample.groupby('gesture')['acc_magnitude'].mean().sort_values()\ngesture_acc.plot(kind='barh', ax=axes[0, 0])\naxes[0, 0].set_title('Mean Acceleration Magnitude by Gesture')\naxes[0, 0].set_xlabel('Acceleration Magnitude')\n\n# Thermopile temperature by gesture\ngesture_thm = train_sample.groupby('gesture')[thm_cols].mean().mean(axis=1).sort_values()\ngesture_thm.plot(kind='barh', ax=axes[0, 1])\naxes[0, 1].set_title('Mean Thermopile Temperature by Gesture')\naxes[0, 1].set_xlabel('Temperature (°C)')\n\n# Sequence length by gesture\nseq_lengths = seq_stats.groupby('gesture')['length'].mean().sort_values()\nseq_lengths.plot(kind='barh', ax=axes[1, 0])\naxes[1, 0].set_title('Mean Sequence Length by Gesture')\naxes[1, 0].set_xlabel('Sequence Length')\n\n# Target vs Non-target comparison\ntarget_comparison = train_sample.groupby('sequence_type').agg({\n    'acc_magnitude': 'mean',\n    'thm_1': 'mean',\n    'thm_2': 'mean',\n    'thm_3': 'mean'\n})\ntarget_comparison.T.plot(kind='bar', ax=axes[1, 1])\naxes[1, 1].set_title('Target vs Non-target Sensor Patterns')\naxes[1, 1].set_ylabel('Mean Value')\naxes[1, 1].legend(['Non-target', 'Target'])\naxes[1, 1].tick_params(axis='x', rotation=45)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:23.178718Z","iopub.execute_input":"2025-07-17T15:25:23.179121Z","iopub.status.idle":"2025-07-17T15:25:24.789290Z","shell.execute_reply.started":"2025-07-17T15:25:23.179086Z","shell.execute_reply":"2025-07-17T15:25:24.788160Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Advanced Feature Engineering","metadata":{}},{"cell_type":"code","source":"def extract_advanced_features(df):\n    \"\"\"Extract comprehensive features from sensor data\"\"\"\n    features = {}\n    \n    # === IMU FEATURES ===\n    imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\n    \n    for col in imu_cols:\n        if col in df.columns:\n            data = df[col].dropna()\n            if len(data) > 0:\n                # Basic statistics\n                features[f'{col}_mean'] = data.mean()\n                features[f'{col}_std'] = data.std()\n                features[f'{col}_min'] = data.min()\n                features[f'{col}_max'] = data.max()\n                features[f'{col}_range'] = data.max() - data.min()\n                features[f'{col}_median'] = data.median()\n                features[f'{col}_q25'] = data.quantile(0.25)\n                features[f'{col}_q75'] = data.quantile(0.75)\n                features[f'{col}_iqr'] = data.quantile(0.75) - data.quantile(0.25)\n                \n                # Advanced statistics\n                features[f'{col}_skew'] = data.skew()\n                features[f'{col}_kurtosis'] = data.kurtosis()\n                features[f'{col}_rms'] = np.sqrt((data**2).mean())\n                features[f'{col}_energy'] = (data**2).sum()\n                \n                # Derivative features\n                if len(data) > 1:\n                    diff = data.diff().dropna()\n                    features[f'{col}_diff_mean'] = diff.mean()\n                    features[f'{col}_diff_std'] = diff.std()\n                    features[f'{col}_diff_max'] = diff.abs().max()\n                    \n                    # Zero crossings\n                    features[f'{col}_zero_crossings'] = ((data.shift(1) * data) < 0).sum()\n    \n    # === COMPOSITE IMU FEATURES ===\n    if all(col in df.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n        acc_mag = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n        features['acc_magnitude_mean'] = acc_mag.mean()\n        features['acc_magnitude_std'] = acc_mag.std()\n        features['acc_magnitude_max'] = acc_mag.max()\n        features['acc_magnitude_energy'] = (acc_mag**2).sum()\n        \n        # Jerk (rate of change of acceleration)\n        if len(acc_mag) > 1:\n            jerk = acc_mag.diff().abs()\n            features['jerk_mean'] = jerk.mean()\n            features['jerk_std'] = jerk.std()\n            features['jerk_max'] = jerk.max()\n    \n    # === THERMOPILE FEATURES ===\n    thm_cols = [f'thm_{i}' for i in range(1, 6)]\n    thm_present = [col for col in thm_cols if col in df.columns]\n    \n    if thm_present:\n        thm_data = df[thm_present]\n        \n        # Individual sensor features\n        for col in thm_present:\n            data = df[col].dropna()\n            if len(data) > 0:\n                features[f'{col}_mean'] = data.mean()\n                features[f'{col}_std'] = data.std()\n                features[f'{col}_min'] = data.min()\n                features[f'{col}_max'] = data.max()\n                features[f'{col}_range'] = data.max() - data.min()\n        \n        # Aggregate thermopile features\n        features['thm_mean_all'] = thm_data.mean().mean()\n        features['thm_std_all'] = thm_data.std().mean()\n        features['thm_max_all'] = thm_data.max().max()\n        features['thm_min_all'] = thm_data.min().min()\n        features['thm_range_all'] = features['thm_max_all'] - features['thm_min_all']\n        \n        # Temperature gradient features\n        if len(thm_present) > 1:\n            thm_corr = thm_data.corr().values\n            features['thm_correlation_mean'] = np.mean(thm_corr[np.triu_indices_from(thm_corr, k=1)])\n    \n    # === TIME-OF-FLIGHT FEATURES ===\n    tof_cols = [col for col in df.columns if col.startswith('tof_')]\n    if tof_cols:\n        tof_data = df[tof_cols].replace(-1, np.nan)\n        \n        # Basic ToF statistics\n        features['tof_mean'] = tof_data.mean().mean()\n        features['tof_std'] = tof_data.std().mean()\n        features['tof_min'] = tof_data.min().min()\n        features['tof_max'] = tof_data.max().max()\n        features['tof_range'] = features['tof_max'] - features['tof_min']\n        \n        # Response rate features\n        total_measurements = len(df) * len(tof_cols)\n        valid_measurements = tof_data.notna().sum().sum()\n        features['tof_response_rate'] = valid_measurements / total_measurements\n        \n        # Spatial features (assuming 8x8 grid per sensor)\n        for sensor_num in range(1, 6):\n            sensor_cols = [col for col in tof_cols if col.startswith(f'tof_{sensor_num}_')]\n            if len(sensor_cols) == 64:  # Full 8x8 grid\n                sensor_data = df[sensor_cols].replace(-1, np.nan)\n                features[f'tof_{sensor_num}_mean'] = sensor_data.mean().mean()\n                features[f'tof_{sensor_num}_response_rate'] = sensor_data.notna().sum().sum() / (len(df) * 64)\n    \n    # === TEMPORAL FEATURES ===\n    features['sequence_length'] = len(df)\n    \n    if 'sequence_counter' in df.columns:\n        features['sequence_duration'] = df['sequence_counter'].max() - df['sequence_counter'].min()\n    \n    # === BEHAVIORAL CONTEXT FEATURES ===\n    if 'behavior' in df.columns:\n        behavior_counts = df['behavior'].value_counts()\n        for behavior, count in behavior_counts.items():\n            features[f'behavior_{behavior}_ratio'] = count / len(df)\n    \n    return features\n\nprint(\"Advanced feature extraction function defined\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:24.790473Z","iopub.execute_input":"2025-07-17T15:25:24.790826Z","iopub.status.idle":"2025-07-17T15:25:24.819516Z","shell.execute_reply.started":"2025-07-17T15:25:24.790775Z","shell.execute_reply":"2025-07-17T15:25:24.818071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Extract features for a sample of sequences\nprint(\"=== FEATURE EXTRACTION ANALYSIS ===\")\n\n# Sample a few sequences for detailed analysis\nsample_sequences = train_sample['sequence_id'].unique()[:20]\nfeature_samples = []\nlabels_samples = []\n\nprint(f\"Extracting features for {len(sample_sequences)} sequences...\")\n\nfor seq_id in sample_sequences:\n    seq_data = train_sample[train_sample['sequence_id'] == seq_id]\n    if len(seq_data) > 0 and 'gesture' in seq_data.columns:\n        features = extract_advanced_features(seq_data)\n        gesture = seq_data['gesture'].iloc[0]\n        \n        feature_samples.append(features)\n        labels_samples.append(gesture)\n\nif feature_samples:\n    features_df = pd.DataFrame(feature_samples)\n    features_df = features_df.fillna(0)\n    \n    print(f\"\\nExtracted {len(features_df.columns)} features:\")\n    print(f\"Feature categories:\")\n    \n    feature_categories = {\n        'IMU': [col for col in features_df.columns if any(imu in col for imu in ['acc_', 'rot_'])],\n        'Thermopile': [col for col in features_df.columns if 'thm_' in col],\n        'ToF': [col for col in features_df.columns if 'tof_' in col],\n        'Temporal': [col for col in features_df.columns if any(temp in col for temp in ['sequence_', 'duration'])],\n        'Composite': [col for col in features_df.columns if any(comp in col for comp in ['magnitude', 'jerk', 'energy'])],\n        'Other': [col for col in features_df.columns if not any(cat in col for cat in ['acc_', 'rot_', 'thm_', 'tof_', 'sequence_', 'duration', 'magnitude', 'jerk', 'energy'])]\n    }\n    \n    for category, cols in feature_categories.items():\n        print(f\"  {category}: {len(cols)} features\")\n    \n    # Feature importance analysis (if we have enough samples)\n    if len(features_df) > 5 and len(set(labels_samples)) > 1:\n        print(\"\\nCalculating feature importance...\")\n        \n        # Standardize features\n        scaler = StandardScaler()\n        features_scaled = scaler.fit_transform(features_df)\n        \n        # Calculate mutual information\n        mi_scores = mutual_info_classif(features_scaled, labels_samples, random_state=1)\n        \n        # Create feature importance dataframe\n        feature_importance = pd.DataFrame({\n            'feature': features_df.columns,\n            'importance': mi_scores\n        }).sort_values('importance', ascending=False)\n        \n        print(\"\\nTop 15 most important features:\")\n        print(feature_importance.head(15))\n        \n        # Plot feature importance\n        plt.figure(figsize=(12, 8))\n        top_features = feature_importance.head(20)\n        plt.barh(range(len(top_features)), top_features['importance'])\n        plt.yticks(range(len(top_features)), top_features['feature'])\n        plt.xlabel('Mutual Information Score')\n        plt.title('Top 20 Feature Importance (Mutual Information)')\n        plt.gca().invert_yaxis()\n        plt.tight_layout()\n        plt.show()\nelse:\n    print(\"No valid sequences found for feature extraction\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:24.820842Z","iopub.execute_input":"2025-07-17T15:25:24.821204Z","iopub.status.idle":"2025-07-17T15:25:28.457848Z","shell.execute_reply.started":"2025-07-17T15:25:24.821181Z","shell.execute_reply":"2025-07-17T15:25:28.456777Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"=== DATA QUALITY ASSESSMENT ===\")\n\n# Check for data quality issues\nquality_report = {\n    'total_rows': len(train_sample),\n    'unique_sequences': train_sample['sequence_id'].nunique(),\n    'unique_subjects': train_sample['subject'].nunique(),\n    'unique_gestures': train_sample['gesture'].nunique(),\n    'missing_values': train_sample.isnull().sum().sum(),\n    'duplicate_rows': train_sample.duplicated().sum()\n}\n\nprint(\"\\nData Quality Summary:\")\nfor key, value in quality_report.items():\n    print(f\"  {key.replace('_', ' ').title()}: {value:,}\")\n\n# Check sequence completeness\nprint(\"\\nSequence Completeness Analysis:\")\nseq_completeness = train_sample.groupby('sequence_id').agg({\n    'sequence_counter': ['min', 'max', 'count'],\n    'gesture': 'nunique',\n    'subject': 'nunique'\n})\n\n# Check for sequences with multiple gestures (should be 1)\nmulti_gesture_seqs = seq_completeness[seq_completeness[('gesture', 'nunique')] > 1]\nprint(f\"Sequences with multiple gestures: {len(multi_gesture_seqs)}\")\n\n# Check for sequences with multiple subjects (should be 1)\nmulti_subject_seqs = seq_completeness[seq_completeness[('subject', 'nunique')] > 1]\nprint(f\"Sequences with multiple subjects: {len(multi_subject_seqs)}\")\n\n# Sensor data availability\nprint(\"\\nSensor Data Availability:\")\nsensor_availability = {\n    'IMU sensors': sum(1 for col in imu_cols if col in train_sample.columns),\n    'Thermopile sensors': sum(1 for col in thm_cols if col in train_sample.columns),\n    'ToF sensors': len([col for col in train_sample.columns if col.startswith('tof_')])\n}\n\nfor sensor_type, count in sensor_availability.items():\n    print(f\"  {sensor_type}: {count} columns\")\n\n# Data range validation\nprint(\"\\nData Range Validation:\")\nif 'acc_magnitude' in train_sample.columns:\n    acc_range = (train_sample['acc_magnitude'].min(), train_sample['acc_magnitude'].max())\n    print(f\"  Acceleration magnitude range: {acc_range[0]:.2f} - {acc_range[1]:.2f}\")\n\nif thm_cols[0] in train_sample.columns:\n    thm_range = (train_sample[thm_cols].min().min(), train_sample[thm_cols].max().max())\n    print(f\"  Thermopile temperature range: {thm_range[0]:.1f}°C - {thm_range[1]:.1f}°C\")\n\nif tof_cols:\n    tof_valid = train_sample[tof_cols].replace(-1, np.nan)\n    tof_range = (tof_valid.min().min(), tof_valid.max().max())\n    print(f\"  ToF sensor range: {tof_range[0]:.0f} - {tof_range[1]:.0f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-17T15:25:28.458928Z","iopub.execute_input":"2025-07-17T15:25:28.459259Z","iopub.status.idle":"2025-07-17T15:25:43.031609Z","shell.execute_reply.started":"2025-07-17T15:25:28.459232Z","shell.execute_reply":"2025-07-17T15:25:43.030343Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}