{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-13T17:30:38.027937Z","iopub.execute_input":"2025-11-13T17:30:38.029615Z","iopub.status.idle":"2025-11-13T17:30:38.075645Z","shell.execute_reply.started":"2025-11-13T17:30:38.029568Z","shell.execute_reply":"2025-11-13T17:30:38.074793Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T17:30:38.076637Z","iopub.execute_input":"2025-11-13T17:30:38.076935Z","iopub.status.idle":"2025-11-13T17:30:38.081313Z","shell.execute_reply.started":"2025-11-13T17:30:38.076904Z","shell.execute_reply":"2025-11-13T17:30:38.080273Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntrain = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\", low_memory=False)\ntest = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\", low_memory=False)\ntrain_demo = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\ntest_demo = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\")\n\nprint(train.shape, test.shape)\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T17:30:38.082898Z","iopub.execute_input":"2025-11-13T17:30:38.083279Z","iopub.status.idle":"2025-11-13T17:31:40.418592Z","shell.execute_reply.started":"2025-11-13T17:30:38.083254Z","shell.execute_reply":"2025-11-13T17:31:40.417682Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom scipy.stats import skew, kurtosis\nfrom scipy.fft import fft\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import f1_score\nimport os\n\n# === CONFIGURATION ===\n# Using the user-provided file paths\nTRAIN_CSV = '/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv'\nTEST_CSV = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'\nDEMOG_TRAIN = '/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv'\nDEMOG_TEST = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv'\n\nFEATURE_OUT = '/kaggle/working/sequence_features.csv'\nRANDOM_SEED = 42\nWINDOW_SIZE = 50 # A tunable hyperparameter\n\n# Define sensor column groups\nACC_COLS = ['acc_x', 'acc_y', 'acc_z']\nROT_COLS = ['rot_w', 'rot_x', 'rot_y', 'rot_z']\nIMU_COLS = ACC_COLS + ROT_COLS\nTHM_COLS = [f'thm_{i}' for i in range(1, 6)]\nTOF_COLS = [f'tof_{i}_v{j}' for i in range(1, 6) for j in range(0, 64)]\n\n\nprint(\"DONE\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T18:58:55.378550Z","iopub.execute_input":"2025-11-13T18:58:55.378934Z","iopub.status.idle":"2025-11-13T18:58:55.402456Z","shell.execute_reply.started":"2025-11-13T18:58:55.378908Z","shell.execute_reply":"2025-11-13T18:58:55.401153Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 1: FEATURE ENGINEERING FUNCTIONS\n# ===============================================\n\ndef extract_imu_features(window_df):\n    \"\"\"Extracts Time-Domain, Frequency-Domain, and statistical IMU features.\"\"\"\n    features = {}\n    \n    # ... (other code for Time-Domain features)\n    \n    # Time-Domain and Statistical Features (CORRECTED SECTION)\n    for col in IMU_COLS:\n        col_prefix = col\n        features[f'{col_prefix}_mean'] = window_df[col].mean()\n        features[f'{col_prefix}_var'] = window_df[col].var()\n        \n        # Explicitly check sample size before calculating skew and kurtosis\n        data = window_df[col].dropna()\n        MIN_SAMPLE_SIZE = 8 # Set minimum required samples (e.g., 8 is a common threshold)\n        \n        if len(data) >= MIN_SAMPLE_SIZE:\n            features[f'{col_prefix}_skew'] = skew(data)\n            features[f'{col_prefix}_kurt'] = kurtosis(data)\n        else:\n            # If sample is too small, set to NaN (which the imputer will handle later)\n            features[f'{col_prefix}_skew'] = np.nan\n            features[f'{col_prefix}_kurt'] = np.nan\n    #Signal Magnitude Area (SMA) and Total Magnitude\n    acc_mag = np.sqrt(np.square(window_df[ACC_COLS]).sum(axis=1))\n    features['acc_SMA'] = acc_mag.sum() / WINDOW_SIZE\n    features['acc_mag_mean'] = acc_mag.mean()\n    features['acc_mag_var'] = acc_mag.var()\n\n    # Cross-Axis Correlation\n    features['acc_corr_xy'] = window_df['acc_x'].corr(window_df['acc_y'])\n    \n    # Frequency-Domain Features (FFT magnitude of first few non-DC components)\n    for col in IMU_COLS:\n        if len(window_df[col].dropna()) > 0:\n            fft_values = np.abs(fft(window_df[col].fillna(window_df[col].mean()).values))\n            for i in range(1, 6):\n                features[f'{col}_fft_mag_{i}'] = fft_values[i]\n        else:\n             for i in range(1, 6):\n                features[f'{col}_fft_mag_{i}'] = np.nan\n            \n    return features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:05:21.619952Z","iopub.execute_input":"2025-11-13T19:05:21.620324Z","iopub.status.idle":"2025-11-13T19:05:21.629965Z","shell.execute_reply.started":"2025-11-13T19:05:21.620300Z","shell.execute_reply":"2025-11-13T19:05:21.629113Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_thermopile_features(window_df):\n    \"\"\"Extracts features from Thermopile data (thm_1 to thm_5), including gradients.\"\"\"\n    features = {}\n\n    # Simple Statistical Features (Average Heat Intensity, Temporal Variance)\n    for col in THM_COLS:\n        col_prefix = col\n        features[f'{col_prefix}_mean'] = window_df[col].mean()\n        features[f'{col_prefix}_var'] = window_df[col].var()\n\n    # Thermopile Gradient Features (Difference between adjacent sensors)\n    for i in range(1, 5):\n        grad_col = f'thm_grad_{i}_{i+1}'\n        gradient = window_df[f'thm_{i}'] - window_df[f'thm_{i+1}']\n        features[f'{grad_col}_mean'] = gradient.mean()\n        features[f'{grad_col}_var'] = gradient.var()\n        \n    return features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:05:30.477463Z","iopub.execute_input":"2025-11-13T19:05:30.477792Z","iopub.status.idle":"2025-11-13T19:05:30.484665Z","shell.execute_reply.started":"2025-11-13T19:05:30.477770Z","shell.execute_reply":"2025-11-13T19:05:30.483577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_tof_features(window_df):\n    \"\"\"Extracts spatial and temporal features from Time-of-Flight (ToF) data.\"\"\"\n    features = {}\n\n    # Replace the 'no sensor response' value (-1) with NaN for statistics\n    tof_data = window_df[TOF_COLS].replace(-1, np.nan)\n    \n    # Overall Proximity Features\n    features['tof_min_proximity'] = tof_data.min().min() # Closest object\n    features['tof_max_proximity'] = tof_data.max().max() # Farthest object\n    \n    # Per-Sensor (5 sensors) Spatial and Temporal Features\n    for i in range(1, 6):\n        sensor_cols = [f'tof_{i}_v{j}' for j in range(0, 64)]\n        tof_sensor_data = tof_data[sensor_cols]\n        prefix = f'tof_{i}'\n\n        # 1. Spatial Statistics\n        spatial_mean_series = tof_sensor_data.mean(axis=1) # Mean distance per row\n        features[f'{prefix}_spatial_mean'] = spatial_mean_series.mean()\n        features[f'{prefix}_spatial_var'] = spatial_mean_series.var()\n        \n        # 2. Number of 'Active' Pixels (Non-negative)\n        active_pixels = (window_df[sensor_cols] != -1).sum(axis=1)\n        features[f'{prefix}_active_pixels_mean'] = active_pixels.mean()\n        features[f'{prefix}_active_pixels_var'] = active_pixels.var()\n\n        # 3. Temporal Slope / Trend\n        avg_proximity_series = spatial_mean_series.dropna()\n        if len(avg_proximity_series) >= 2:\n            # Calculate temporal slope (change in average distance over time)\n            slope = np.polyfit(np.arange(len(avg_proximity_series)), avg_proximity_series, 1)[0]\n            features[f'{prefix}_temporal_slope'] = slope\n        else:\n            features[f'{prefix}_temporal_slope'] = np.nan\n\n    return features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:05:33.458309Z","iopub.execute_input":"2025-11-13T19:05:33.458633Z","iopub.status.idle":"2025-11-13T19:05:33.467092Z","shell.execute_reply.started":"2025-11-13T19:05:33.458611Z","shell.execute_reply":"2025-11-13T19:05:33.465898Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 2: DATA PIPELINE (Windowing, Extraction, Merging)\n# ===============================================\n\ndef run_feature_pipeline(ts_path, demog_path, is_train=True):\n    \"\"\"Loads, processes time-series data, and merges with demographics.\"\"\"\n    print(f\"--- Running Pipeline for {'Train' if is_train else 'Test'} Data ---\")\n    \n    # Load Time-Series and Demographics\n    df_ts = pd.read_csv(ts_path)\n    df_demog = pd.read_csv(demog_path)\n    \n    features_list = []\n    \n    # Iterate through each unique sequence\n    sequence_groups = df_ts.groupby('sequence_id')\n    for seq_id, group in sequence_groups:\n        \n        # --- Windowing Strategy ---\n        # For a full sequence (Transition, Pause, Gesture), the critical event \n        # is the Gesture, which is at the end of the sequence. We use the last \n        # WINDOW_SIZE rows to capture this final motion.\n        window_df = group.tail(WINDOW_SIZE).copy()\n        \n        if len(window_df) < WINDOW_SIZE:\n            # Skip short sequences that can't form a reliable feature vector\n            continue\n            \n        # 1. Feature Extraction\n        features = {}\n        features.update(extract_imu_features(window_df))\n        features.update(extract_thermopile_features(window_df))\n        features.update(extract_tof_features(window_df))\n        \n        # 2. Add identifying and target columns\n        features['sequence_id'] = seq_id\n        features['subject'] = window_df['subject'].iloc[0]\n        if is_train:\n            # The gesture column is only present in the training set\n            features['gesture'] = window_df['gesture'].iloc[-1] \n        \n        features_list.append(features)\n        \n    df_features = pd.DataFrame(features_list)\n    \n    # 3. Merge with Demographics (Static Features Fusion)\n    df_final = pd.merge(df_features, df_demog, on='subject', how='left')\n    \n    print(f\"Generated {len(df_final)} feature vectors. Total features: {len(df_final.columns)}\")\n    return df_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:05:36.231060Z","iopub.execute_input":"2025-11-13T19:05:36.231344Z","iopub.status.idle":"2025-11-13T19:05:36.240000Z","shell.execute_reply.started":"2025-11-13T19:05:36.231325Z","shell.execute_reply":"2025-11-13T19:05:36.238968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Execute pipeline for both datasets\ndf_train_features = run_feature_pipeline(TRAIN_CSV, DEMOG_TRAIN, is_train=True)\ndf_test_features = run_feature_pipeline(TEST_CSV, DEMOG_TEST, is_train=False)\n\n# Save the features (optional, for debugging/reuse)\ndf_train_features.to_csv(FEATURE_OUT.replace(\".csv\", \"_train.csv\"), index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:05:38.883707Z","iopub.execute_input":"2025-11-13T19:05:38.884059Z","iopub.status.idle":"2025-11-13T19:09:16.162928Z","shell.execute_reply.started":"2025-11-13T19:05:38.884034Z","shell.execute_reply":"2025-11-13T19:09:16.161546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# STEP 3: MODELING PIPELINE (Imputation, Scaling, Training)\n# ===============================================\n\ndef train_and_evaluate_model(df_train_features, df_test_features, random_seed):\n    \"\"\"Prepares data, trains Random Forest model, and prepares submission.\"\"\"\n    \n    # --- Data Preparation ---\n    # 1. Separate Features (X) and Target (y)\n    X_train = df_train_features.drop(columns=['sequence_id', 'subject', 'gesture'])\n    y_train = df_train_features['gesture']\n    \n    # Align test features to match training feature columns\n    X_test_seq_id = df_test_features['sequence_id']\n    X_test = df_test_features.drop(columns=['sequence_id', 'subject'])\n    \n    # 2. Define the Model Pipeline\n    IMPUTATION_STEP = ('imputer', SimpleImputer(strategy='median')) # Handles NaNs from feature engineering\n    \n    # Random Forest (Tree-based model, handles complexity, with class weighting)\n    rf_pipe = Pipeline([\n        IMPUTATION_STEP, \n        ('scaler', StandardScaler()), # Scaling helps with consistent feature contribution\n        ('clf', RandomForestClassifier(\n            n_estimators=500, # Increased n_estimators for better performance\n            random_state=random_seed,\n            class_weight='balanced', # Crucial for handling imbalanced BFRB/Non-BFRB classes\n            n_jobs=-1 # Use all cores\n        ))\n    ])\n    \n    # --- Training and Prediction ---\n    print(\"\\nStarting Model Training...\")\n    rf_pipe.fit(X_train, y_train)\n    print(\"Training Complete.\")\n    \n    # Predict on Test Data\n    y_pred_test = rf_pipe.predict(X_test)\n    \n    # --- Evaluation (Conceptual: Needs cross-validation for real score) ---\n    # For a quick check, predict on training data:\n    y_pred_train = rf_pipe.predict(X_train)\n    macro_f1_train = f1_score(y_train, y_pred_train, average='macro')\n    print(f\"Training Macro F1 Score (Validation Check): {macro_f1_train:.4f}\")\n    \n    # --- Submission File Generation ---\n    df_submission = pd.DataFrame({\n        'sequence_id': X_test_seq_id,\n        'gesture': y_pred_test\n    })\n    \n    # Final Output file path for Kaggle submission\n    submission_path = os.path.join('/kaggle/working', 'submission.csv')\n    df_submission.to_csv(submission_path, index=False)\n    print(f\"\\nSubmission file created at: {submission_path}\")\n    print(df_submission.head())\n\n# Execute the final modeling step\ntrain_and_evaluate_model(df_train_features, df_test_features, RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T19:11:45.646697Z","iopub.execute_input":"2025-11-13T19:11:45.647141Z","iopub.status.idle":"2025-11-13T19:12:02.575605Z","shell.execute_reply.started":"2025-11-13T19:11:45.647109Z","shell.execute_reply":"2025-11-13T19:12:02.574514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier # NEW\nfrom sklearn.neighbors import KNeighborsClassifier # NEW\nfrom sklearn.ensemble import RandomForestClassifier, VotingClassifier # UPDATED\nfrom sklearn.metrics import f1_score, confusion_matrix\nimport os\n\n# --- Imbalance Handling Note ---\n# Class Weighting (class_weight='balanced') is used for LR, DT, and RF as it's built-in \n# and often preferred inside pipelines. SMOTE, though effective, requires an external \n# library (imblearn) and must be applied BEFORE scaling, which complicates the pipeline.\n\ndef train_and_evaluate_model(df_train_features, df_test_features, random_seed):\n    \"\"\"Defines, trains, and evaluates all baseline, tree-based, and ensemble models.\"\"\"\n    \n    print(\"\\nStarting Model Development and Evaluation...\")\n    \n    # --- Data Preparation ---\n    X_train_full = df_train_features.drop(columns=['sequence_id', 'subject', 'gesture'])\n    y_train_full = df_train_features['gesture']\n    \n    # Use a split of the TRAIN data for internal evaluation (Macro F1)\n    X_train, X_val, y_train, y_val = train_test_split(\n        X_train_full, y_train_full, test_size=0.3, random_state=random_seed, stratify=y_train_full\n    )\n    \n    # Prepare Test Data for final predictions\n    X_test_seq_id = df_test_features['sequence_id']\n    X_test = df_test_features.drop(columns=['sequence_id', 'subject'])\n    \n    # --- Pipeline Components ---\n    IMPUTATION_STEP = ('imputer', SimpleImputer(strategy='median'))\n    SCALING_STEP = ('scaler', StandardScaler())\n    \n    \n    # --- 1. Baseline Models ---\n    \n    # 1A. Logistic Regression (Imbalanced handled by class_weight)\n    lr_pipe = Pipeline([\n        IMPUTATION_STEP, SCALING_STEP,\n        ('clf', LogisticRegression(random_state=random_seed, solver='liblinear', multi_class='ovr', class_weight='balanced'))\n    ])\n    \n    # 1B. k-Nearest Neighbors (No inherent imbalance handling, scaling is crucial)\n    knn_pipe = Pipeline([\n        IMPUTATION_STEP, SCALING_STEP,\n        ('clf', KNeighborsClassifier(n_neighbors=5)) # n_neighbors is tunable\n    ])\n    \n    \n    # --- 2. Tree-based Models ---\n    \n    # 2A. Decision Tree (Imbalanced handled by class_weight)\n    dt_pipe = Pipeline([\n        IMPUTATION_STEP,\n        ('clf', DecisionTreeClassifier(random_state=random_seed, class_weight='balanced'))\n    ])\n    \n    # 2B. Random Forest (Imbalanced handled by class_weight)\n    rf_pipe = Pipeline([\n        IMPUTATION_STEP,\n        ('clf', RandomForestClassifier(n_estimators=500, random_state=random_seed, class_weight='balanced', n_jobs=-1))\n    ])\n    \n    \n    # --- 3. Ensemble Model: Voting Classifier ---\n    \n    # Use the best performing and most diverse base models: LR, RF, and DT.\n    # The 'named_estimators' must use the name of the step defined above.\n    \n    \n    # Wrap the ensemble in a pipeline for consistent data preprocessing\n    voting_pipe = VotingClassifier(\n        estimators=[\n            ('lr', lr_pipe),\n            ('rf', rf_pipe),\n            ('dt', dt_pipe)\n        ],\n        voting='soft', \n        n_jobs=-1\n    )\n    \n    \n    # --- 4. Training and Evaluation Loop ---\n    \n    models = {\n        \"Logistic Regression\": lr_pipe,\n        \"k-Nearest Neighbors\": knn_pipe,\n        \"Decision Tree\": dt_pipe,\n        \"Random Forest\": rf_pipe,\n        \"Voting Classifier\": voting_pipe\n    }\n    \n    best_model_name = \"\"\n    best_f1 = -1.0\n    \n    results = {}\n    for name, pipe in models.items():\n        print(f\"\\nTraining {name}...\")\n        pipe.fit(X_train, y_train)\n        y_pred_val = pipe.predict(X_val)\n        \n        macro_f1 = f1_score(y_val, y_pred_val, average='macro')\n        results[name] = macro_f1\n        print(f\"{name} Macro F1 Score (Validation): {macro_f1:.4f}\")\n\n        if macro_f1 > best_f1:\n            best_f1 = macro_f1\n            best_model_name = name\n            \n    # --- 5. Final Prediction using the BEST model ---\n    \n    final_model = models[best_model_name]\n    \n    # Retrain the best model on the FULL training set for best performance\n    print(f\"\\nRetraining BEST model ({best_model_name}) on full training data...\")\n    final_model.fit(X_train_full, y_train_full)\n    \n    y_pred_test = final_model.predict(X_test)\n    \n    # --- Submission File Generation ---\n    df_submission = pd.DataFrame({\n        'sequence_id': X_test_seq_id,\n        'gesture': y_pred_test\n    })\n    \n    submission_path = os.path.join('/kaggle/working', 'submission_ensemble.csv')\n    df_submission.to_csv(submission_path, index=False)\n    \n    print(\"\\n--- Summary of Validation F1 Scores ---\")\n    for name, f1 in results.items():\n         print(f\"{name}: {f1:.4f}\")\n         \n    print(f\"\\nFinal prediction made using: {best_model_name}\")\n    print(f\"Submission file created at: {submission_path}\")\n    print(df_submission.head())\n    \n# Execute the final modeling step\ntrain_and_evaluate_model(df_train_features, df_test_features, RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T20:11:33.821889Z","iopub.execute_input":"2025-11-13T20:11:33.824967Z","iopub.status.idle":"2025-11-13T20:12:28.697702Z","shell.execute_reply.started":"2025-11-13T20:11:33.824932Z","shell.execute_reply":"2025-11-13T20:12:28.696733Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create a dtype map to save memory:\n# Most sensor data can be safely cast to float32.\nDTYPE_MAP = {\n    'row_id': 'object',\n    'sequence_id': 'object',\n    'sequence_counter': 'int16',\n    'subject': 'object',\n    # IMU & THM: Use float32\n    **{col: 'float32' for col in ACC_COLS + ROT_COLS + THM_COLS},\n    # ToF: They are uncalibrated sensor values (0-254) but loaded as float/int. \n    # Use float32 to be safe with operations.\n    **{col: 'float32' for col in TOF_COLS}, \n    # Target columns (for train only)\n    'sequence_type': 'object', \n    'gesture': 'object', \n    'orientation': 'object',\n    'behavior': 'object'\n}\n\n\ndef run_feature_pipeline_optimized(ts_path, demog_path, is_train=True):\n    \"\"\"Loads data with optimized dtypes, processes, and merges with demographics.\"\"\"\n    print(f\"--- Running Optimized Pipeline for {'Train' if is_train else 'Test'} Data ---\")\n    \n    # Load Time-Series with optimized dtypes\n    # We load only the columns we need to save even more memory\n    cols_to_load = ['sequence_id', 'subject', 'sequence_counter'] + IMU_COLS + THM_COLS + TOF_COLS\n    if is_train:\n         cols_to_load += ['gesture']\n    \n    # Filter DTYPE_MAP to only include columns being loaded\n    dtype_filter = {k: v for k, v in DTYPE_MAP.items() if k in cols_to_load}\n    \n    # Load the time-series file with explicit dtypes\n    df_ts = pd.read_csv(ts_path, dtype=dtype_filter, usecols=cols_to_load)\n    df_demog = pd.read_csv(demog_path)\n    \n    # ... (rest of the run_feature_pipeline function remains the same)\n    \n    features_list = []\n    \n    # Iterate through each unique sequence\n    sequence_groups = df_ts.groupby('sequence_id')\n    for seq_id, group in sequence_groups:\n        # ... (feature extraction code remains the same)\n        window_df = group.tail(WINDOW_SIZE).copy()\n        \n        if len(window_df) < WINDOW_SIZE:\n            continue\n            \n        features = {}\n        features.update(extract_imu_features(window_df))\n        features.update(extract_thermopile_features(window_df))\n        features.update(extract_tof_features(window_df))\n        \n        features['sequence_id'] = seq_id\n        features['subject'] = window_df['subject'].iloc[0]\n        if is_train:\n            features['gesture'] = window_df['gesture'].iloc[-1] \n        \n        features_list.append(features)\n        \n    df_features = pd.DataFrame(features_list)\n    \n    # Crucial Memory Step: Explicitly delete large dataframe after features are extracted\n    del df_ts\n    \n    # Merge with Demographics\n    df_final = pd.merge(df_features, df_demog, on='subject', how='left')\n    \n    print(f\"Generated {len(df_final)} feature vectors. Total features: {len(df_final.columns)}\")\n    return df_final","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T20:21:46.695188Z","iopub.execute_input":"2025-11-13T20:21:46.697239Z","iopub.status.idle":"2025-11-13T20:21:46.711127Z","shell.execute_reply.started":"2025-11-13T20:21:46.697192Z","shell.execute_reply":"2025-11-13T20:21:46.710001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Execute pipeline for both datasets using the optimized function\ndf_train_features = run_feature_pipeline_optimized(TRAIN_CSV, DEMOG_TRAIN, is_train=True)\ndf_test_features = run_feature_pipeline_optimized(TEST_CSV, DEMOG_TEST, is_train=False)\n\n# Save the features\n# Saving to disk ensures the memory is cleared and the feature set is usable even if later steps fail.\ndf_train_features.to_csv(FEATURE_OUT.replace(\".csv\", \"_train.csv\"), index=False)\ndf_test_features.to_csv(FEATURE_OUT.replace(\".csv\", \"_test.csv\"), index=False)\n\n# Execute the final modeling step (assuming train_and_evaluate_model is defined correctly)\ntrain_and_evaluate_model(df_train_features, df_test_features, RANDOM_SEED)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T20:23:27.012473Z","iopub.execute_input":"2025-11-13T20:23:27.012794Z","iopub.status.idle":"2025-11-13T20:28:18.370557Z","shell.execute_reply.started":"2025-11-13T20:23:27.012770Z","shell.execute_reply":"2025-11-13T20:28:18.369380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"h=pd.read_csv('/kaggle/working/submission.csv')\nprint(h)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-13T20:53:17.755650Z","iopub.execute_input":"2025-11-13T20:53:17.756135Z","iopub.status.idle":"2025-11-13T20:53:17.775491Z","shell.execute_reply.started":"2025-11-13T20:53:17.756102Z","shell.execute_reply":"2025-11-13T20:53:17.774389Z"}},"outputs":[],"execution_count":null}]}