{"cells":[{"cell_type":"markdown","metadata":{},"source":"# CMI Advanced Physics Model V5 - Target: 87%+ Validation Accuracy\n\n**Major Improvements for 87%+ Target:**\n1. **Advanced Feature Engineering**: FFT, PSD, statistical features (skewness, kurtosis, entropy)\n2. **Time Series Features**: Auto-correlation, cross-correlation, lag features\n3. **Sensor Interaction Features**: Cross-sensor relationships and ratios\n4. **Enhanced Model Architecture**: CatBoost + deeper LightGBM + XGBoost ensemble\n5. **Data Augmentation**: SMOTE for minority classes + time series augmentation\n6. **Hyperparameter Optimization**: Optuna for automated tuning\n7. **Improved Cross-Validation**: GroupKFold for better generalization\n\n**Memory Optimization**: Efficient processing with garbage collection and batch processing"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import numpy as np\nimport pandas as pd\nimport pickle\nfrom pathlib import Path\nimport gc\nimport sys\nimport warnings\nfrom scipy import stats, signal\nfrom scipy.fft import fft, fftfreq\nfrom sklearn.preprocessing import StandardScaler, RobustScaler\nimport optuna\nfrom tqdm import tqdm\nimport joblib\n\nwarnings.filterwarnings(\"ignore\")\n\n# Add path for inference server\nsys.path.append(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")\n\nprint(\"Advanced libraries imported successfully\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Import inference server\nfrom kaggle_evaluation.cmi_inference_server import CMIInferenceServer\n\nprint(\"CMI Inference Server imported\")"},{"cell_type":"markdown","metadata":{},"source":"## Advanced Physics-Based Feature Engineering"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def remove_gravity_from_acc(df_seq, alpha=0.8):\n    \"\"\"重力除去加速度の計算（トップ解法の必須要素）\"\"\"\n    acc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    acc_data = df_seq[acc_cols].values\n\n    # Low-pass filter for gravity estimation\n    gravity = np.zeros_like(acc_data)\n    gravity[0] = acc_data[0]\n\n    for i in range(1, len(acc_data)):\n        gravity[i] = alpha * gravity[i - 1] + (1 - alpha) * acc_data[i]\n\n    # Linear acceleration = total - gravity\n    linear_acc = acc_data - gravity\n    return linear_acc, gravity\n\n\ndef calculate_angular_velocity_from_quat(df_seq):\n    \"\"\"クォータニオンから角速度を計算（0.852スコア達成手法）\"\"\"\n    quat_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\n    quat = df_seq[quat_cols].values\n\n    # Quaternion difference with proper normalization\n    quat_norm = quat / (np.linalg.norm(quat, axis=1, keepdims=True) + 1e-8)\n    quat_diff = np.diff(quat_norm, axis=0)\n\n    # Angular velocity approximation with improved calculation\n    angular_vel = np.zeros((len(df_seq), 3))\n    if len(quat_diff) > 0:\n        angular_vel[1:, 0] = 2 * quat_diff[:, 0]\n        angular_vel[1:, 1] = 2 * quat_diff[:, 1]\n        angular_vel[1:, 2] = 2 * quat_diff[:, 2]\n\n    return angular_vel\n\n\ndef calculate_jerk_and_snap(df_seq):\n    \"\"\"ジャーク（加速度変化率）とスナップ（ジャーク変化率）の計算\"\"\"\n    acc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    acc_data = df_seq[acc_cols].values\n\n    # Jerk = derivative of acceleration\n    jerk = np.zeros_like(acc_data)\n    if len(acc_data) > 1:\n        jerk[1:] = np.diff(acc_data, axis=0)\n\n    # Snap = derivative of jerk\n    snap = np.zeros_like(acc_data)\n    if len(jerk) > 1:\n        snap[1:] = np.diff(jerk, axis=0)\n\n    return jerk, snap"},{"cell_type":"markdown","metadata":{},"source":"## Advanced Signal Processing Features"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def extract_frequency_features(signal_data, sampling_rate=50):\n    \"\"\"周波数領域特徴量の抽出（FFT、パワースペクトル密度）\"\"\"\n    features = {}\n    \n    if len(signal_data) < 4:\n        return {f\"freq_{key}\": 0 for key in ['dominant', 'power_total', 'spectral_entropy', 'spectral_centroid']}\n    \n    try:\n        # FFT\n        fft_values = np.abs(fft(signal_data))\n        freqs = fftfreq(len(signal_data), 1/sampling_rate)\n        \n        # Power spectral density\n        psd = fft_values ** 2\n        \n        # Dominant frequency\n        dominant_freq_idx = np.argmax(psd[1:len(psd)//2]) + 1\n        features['freq_dominant'] = abs(freqs[dominant_freq_idx])\n        \n        # Total spectral power\n        features['freq_power_total'] = np.sum(psd)\n        \n        # Spectral entropy\n        psd_norm = psd / (np.sum(psd) + 1e-8)\n        features['freq_spectral_entropy'] = -np.sum(psd_norm * np.log(psd_norm + 1e-8))\n        \n        # Spectral centroid\n        features['freq_spectral_centroid'] = np.sum(freqs[:len(psd)] * psd) / (np.sum(psd) + 1e-8)\n        \n    except Exception as e:\n        # Fallback values\n        features = {'freq_dominant': 0, 'freq_power_total': 0, \n                   'freq_spectral_entropy': 0, 'freq_spectral_centroid': 0}\n    \n    return features\n\n\ndef extract_statistical_features(signal_data):\n    \"\"\"高度な統計的特徴量（歪度、尖度、エントロピー）\"\"\"\n    features = {}\n    \n    if len(signal_data) == 0:\n        return {f\"stat_{key}\": 0 for key in ['skewness', 'kurtosis', 'entropy', 'iqr', 'rms']}\n    \n    try:\n        # 基本統計量\n        features['stat_skewness'] = stats.skew(signal_data)\n        features['stat_kurtosis'] = stats.kurtosis(signal_data)\n        \n        # エントロピー\n        hist, _ = np.histogram(signal_data, bins=10)\n        hist_norm = hist / (np.sum(hist) + 1e-8)\n        features['stat_entropy'] = -np.sum(hist_norm * np.log(hist_norm + 1e-8))\n        \n        # 四分位範囲\n        features['stat_iqr'] = np.percentile(signal_data, 75) - np.percentile(signal_data, 25)\n        \n        # RMS (Root Mean Square)\n        features['stat_rms'] = np.sqrt(np.mean(signal_data**2))\n        \n    except Exception as e:\n        # Fallback values\n        features = {'stat_skewness': 0, 'stat_kurtosis': 0, 'stat_entropy': 0, \n                   'stat_iqr': 0, 'stat_rms': 0}\n    \n    return features\n\n\ndef extract_time_series_features(signal_data, max_lag=10):\n    \"\"\"時系列特徴量（自己相関、トレンド）\"\"\"\n    features = {}\n    \n    if len(signal_data) < max_lag + 1:\n        return {f\"ts_{key}\": 0 for key in ['autocorr_1', 'autocorr_5', 'trend', 'seasonality']}\n    \n    try:\n        # Auto-correlation at lag 1 and 5\n        features['ts_autocorr_1'] = np.corrcoef(signal_data[:-1], signal_data[1:])[0, 1] if len(signal_data) > 1 else 0\n        if len(signal_data) > 5:\n            features['ts_autocorr_5'] = np.corrcoef(signal_data[:-5], signal_data[5:])[0, 1]\n        else:\n            features['ts_autocorr_5'] = 0\n        \n        # Linear trend (slope of linear regression)\n        x = np.arange(len(signal_data))\n        slope, _, _, _, _ = stats.linregress(x, signal_data)\n        features['ts_trend'] = slope\n        \n        # Seasonality detection (simplified)\n        if len(signal_data) > 20:\n            # Simple seasonality measure using autocorrelation at period/4\n            period = min(len(signal_data) // 4, 10)\n            if period > 0:\n                features['ts_seasonality'] = np.corrcoef(signal_data[:-period], signal_data[period:])[0, 1]\n            else:\n                features['ts_seasonality'] = 0\n        else:\n            features['ts_seasonality'] = 0\n            \n    except Exception as e:\n        features = {'ts_autocorr_1': 0, 'ts_autocorr_5': 0, 'ts_trend': 0, 'ts_seasonality': 0}\n    \n    return features"},{"cell_type":"markdown","metadata":{},"source":"## Enhanced ToF and Sensor Interaction Features"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def extract_advanced_tof_features(df_seq):\n    \"\"\"ToFセンサーの高度な特徴量抽出\"\"\"\n    tof_features = {}\n    \n    for sensor_id in range(1, 6):\n        pixel_cols = [f\"tof_{sensor_id}_v{p}\" for p in range(64)]\n        if all(col in df_seq.columns for col in pixel_cols):\n            tof_data = df_seq[pixel_cols].replace(-1, np.nan)\n            \n            # 基本統計量\n            tof_features[f\"tof_{sensor_id}_mean\"] = tof_data.mean(axis=1).mean()\n            tof_features[f\"tof_{sensor_id}_std\"] = tof_data.std(axis=1).mean()\n            tof_features[f\"tof_{sensor_id}_min\"] = tof_data.min(axis=1).mean()\n            tof_features[f\"tof_{sensor_id}_max\"] = tof_data.max(axis=1).mean()\n            \n            # 高度な統計量\n            tof_values = tof_data.values.flatten()\n            tof_values = tof_values[~np.isnan(tof_values)]\n            \n            if len(tof_values) > 0:\n                tof_features[f\"tof_{sensor_id}_skew\"] = stats.skew(tof_values)\n                tof_features[f\"tof_{sensor_id}_kurtosis\"] = stats.kurtosis(tof_values)\n                tof_features[f\"tof_{sensor_id}_range\"] = np.ptp(tof_values)\n                \n                # 空間的特徴（8x8グリッドの特性）\n                tof_matrix = tof_data.mean(axis=0).values.reshape(8, 8)\n                if not np.all(np.isnan(tof_matrix)):\n                    # 中心と端の差\n                    center_val = np.nanmean(tof_matrix[3:5, 3:5])\n                    edge_val = np.nanmean([tof_matrix[0, :], tof_matrix[-1, :], \n                                          tof_matrix[:, 0], tof_matrix[:, -1]])\n                    tof_features[f\"tof_{sensor_id}_center_edge_diff\"] = center_val - edge_val\n                else:\n                    tof_features[f\"tof_{sensor_id}_center_edge_diff\"] = 0\n            else:\n                tof_features[f\"tof_{sensor_id}_skew\"] = 0\n                tof_features[f\"tof_{sensor_id}_kurtosis\"] = 0\n                tof_features[f\"tof_{sensor_id}_range\"] = 0\n                tof_features[f\"tof_{sensor_id}_center_edge_diff\"] = 0\n        else:\n            # Default values when sensor data is missing\n            for suffix in ['mean', 'std', 'min', 'max', 'skew', 'kurtosis', 'range', 'center_edge_diff']:\n                tof_features[f\"tof_{sensor_id}_{suffix}\"] = 0\n    \n    return tof_features\n\n\ndef extract_sensor_interaction_features(df_seq):\n    \"\"\"センサー間の相互作用特徴量\"\"\"\n    features = {}\n    \n    # 加速度成分間の相関\n    acc_cols = [\"acc_x\", \"acc_y\", \"acc_z\"]\n    if all(col in df_seq.columns for col in acc_cols):\n        acc_data = df_seq[acc_cols]\n        \n        try:\n            corr_matrix = acc_data.corr()\n            features['acc_xy_corr'] = corr_matrix.loc['acc_x', 'acc_y']\n            features['acc_xz_corr'] = corr_matrix.loc['acc_x', 'acc_z']\n            features['acc_yz_corr'] = corr_matrix.loc['acc_y', 'acc_z']\n        except:\n            features['acc_xy_corr'] = 0\n            features['acc_xz_corr'] = 0\n            features['acc_yz_corr'] = 0\n    \n    # 回転と加速度の関係\n    rot_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\n    if all(col in df_seq.columns for col in acc_cols + rot_cols):\n        try:\n            acc_mag = np.sqrt(df_seq['acc_x']**2 + df_seq['acc_y']**2 + df_seq['acc_z']**2)\n            rot_mag = np.sqrt(df_seq['rot_x']**2 + df_seq['rot_y']**2 + df_seq['rot_z']**2)\n            \n            features['acc_rot_corr'] = np.corrcoef(acc_mag, rot_mag)[0, 1] if len(acc_mag) > 1 else 0\n        except:\n            features['acc_rot_corr'] = 0\n    \n    # 温度センサー間の関係\n    temp_cols = [f\"thm_{i}\" for i in range(1, 6)]\n    available_temp_cols = [col for col in temp_cols if col in df_seq.columns]\n    \n    if len(available_temp_cols) > 1:\n        temp_data = df_seq[available_temp_cols]\n        features['temp_range'] = temp_data.max().max() - temp_data.min().min()\n        features['temp_var'] = temp_data.var(axis=1).mean()\n    else:\n        features['temp_range'] = 0\n        features['temp_var'] = 0\n    \n    return features"},{"cell_type":"markdown","metadata":{},"source":"## Comprehensive Feature Extraction"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def create_advanced_physics_features(df_seq):\n    \"\"\"包括的な高度特徴量生成（87%精度達成のための統合特徴量）\"\"\"\n    features = {}\n    \n    # 1. 重力除去加速度の高度な解析\n    linear_acc, gravity = remove_gravity_from_acc(df_seq)\n    \n    # Linear acceleration features\n    for i, axis in enumerate(['x', 'y', 'z']):\n        signal = linear_acc[:, i]\n        \n        # 基本統計量\n        features[f\"linear_acc_{axis}_mean\"] = np.mean(signal)\n        features[f\"linear_acc_{axis}_std\"] = np.std(signal)\n        features[f\"linear_acc_{axis}_max\"] = np.max(signal)\n        features[f\"linear_acc_{axis}_min\"] = np.min(signal)\n        \n        # 高度な統計特徴\n        stat_features = extract_statistical_features(signal)\n        for key, val in stat_features.items():\n            features[f\"linear_acc_{axis}_{key}\"] = val\n        \n        # 周波数特徴\n        freq_features = extract_frequency_features(signal)\n        for key, val in freq_features.items():\n            features[f\"linear_acc_{axis}_{key}\"] = val\n        \n        # 時系列特徴\n        ts_features = extract_time_series_features(signal)\n        for key, val in ts_features.items():\n            features[f\"linear_acc_{axis}_{key}\"] = val\n    \n    # Linear acceleration magnitude\n    linear_acc_mag = np.sqrt(np.sum(linear_acc**2, axis=1))\n    features[\"linear_acc_mag_mean\"] = np.mean(linear_acc_mag)\n    features[\"linear_acc_mag_std\"] = np.std(linear_acc_mag)\n    features[\"linear_acc_mag_max\"] = np.max(linear_acc_mag)\n    \n    # Gravity features\n    gravity_mag = np.sqrt(np.sum(gravity**2, axis=1))\n    features[\"gravity_mag_mean\"] = np.mean(gravity_mag)\n    features[\"gravity_mag_std\"] = np.std(gravity_mag)\n    \n    # 2. 角速度の高度な解析\n    angular_vel = calculate_angular_velocity_from_quat(df_seq)\n    \n    for i, axis in enumerate(['x', 'y', 'z']):\n        signal = angular_vel[:, i]\n        features[f\"angular_vel_{axis}_mean\"] = np.mean(signal)\n        features[f\"angular_vel_{axis}_std\"] = np.std(signal)\n        \n        # 統計・周波数・時系列特徴を追加\n        for feat_dict, prefix in [(extract_statistical_features(signal), 'stat'),\n                                 (extract_frequency_features(signal), 'freq'),\n                                 (extract_time_series_features(signal), 'ts')]:\n            for key, val in feat_dict.items():\n                features[f\"angular_vel_{axis}_{key}\"] = val\n    \n    angular_vel_mag = np.sqrt(np.sum(angular_vel**2, axis=1))\n    features[\"angular_vel_mag_mean\"] = np.mean(angular_vel_mag)\n    features[\"angular_vel_mag_std\"] = np.std(angular_vel_mag)\n    \n    # 3. ジャークとスナップの解析\n    jerk, snap = calculate_jerk_and_snap(df_seq)\n    \n    # Jerk features\n    jerk_mag = np.sqrt(np.sum(jerk**2, axis=1))\n    features[\"jerk_mag_mean\"] = np.mean(jerk_mag)\n    features[\"jerk_mag_std\"] = np.std(jerk_mag)\n    features[\"jerk_mag_max\"] = np.max(jerk_mag)\n    \n    # Snap features\n    snap_mag = np.sqrt(np.sum(snap**2, axis=1))\n    features[\"snap_mag_mean\"] = np.mean(snap_mag)\n    features[\"snap_mag_std\"] = np.std(snap_mag)\n    \n    # 4. 基本加速度の全特徴\n    acc_mag = np.sqrt(df_seq[\"acc_x\"] ** 2 + df_seq[\"acc_y\"] ** 2 + df_seq[\"acc_z\"] ** 2)\n    \n    # 基本統計量\n    features[\"acc_mag_mean\"] = np.mean(acc_mag)\n    features[\"acc_mag_std\"] = np.std(acc_mag)\n    features[\"acc_mag_max\"] = np.max(acc_mag)\n    features[\"acc_mag_min\"] = np.min(acc_mag)\n    \n    # 高度な特徴\n    for feat_dict, prefix in [(extract_statistical_features(acc_mag), 'stat'),\n                             (extract_frequency_features(acc_mag), 'freq'),\n                             (extract_time_series_features(acc_mag), 'ts')]:\n        for key, val in feat_dict.items():\n            features[f\"acc_mag_{key}\"] = val\n    \n    # 5. 回転特徴量の拡張\n    rot_cols = [\"rot_x\", \"rot_y\", \"rot_z\", \"rot_w\"]\n    for col in rot_cols:\n        if col in df_seq.columns:\n            signal = df_seq[col].values\n            features[f\"{col}_mean\"] = np.mean(signal)\n            features[f\"{col}_std\"] = np.std(signal)\n            \n            # 統計特徴を追加\n            stat_features = extract_statistical_features(signal)\n            for key, val in stat_features.items():\n                features[f\"{col}_{key}\"] = val\n    \n    # 6. 温度センサーの高度な解析\n    temp_cols = [f\"thm_{i}\" for i in range(1, 6)]\n    available_temp_cols = [col for col in temp_cols if col in df_seq.columns]\n    \n    if available_temp_cols:\n        temp_data = df_seq[available_temp_cols]\n        features[\"temp_mean\"] = temp_data.mean().mean()\n        features[\"temp_std\"] = temp_data.std().mean()\n        features[\"temp_max\"] = temp_data.max().max()\n        features[\"temp_min\"] = temp_data.min().min()\n        \n        # 温度の時系列特徴\n        temp_mean_series = temp_data.mean(axis=1)\n        temp_ts_features = extract_time_series_features(temp_mean_series)\n        for key, val in temp_ts_features.items():\n            features[f\"temp_{key}\"] = val\n    else:\n        # Default values\n        for suffix in ['mean', 'std', 'max', 'min', 'ts_autocorr_1', 'ts_autocorr_5', 'ts_trend', 'ts_seasonality']:\n            features[f\"temp_{suffix}\"] = 0\n    \n    # 7. ToFセンサーの高度な特徴量\n    tof_features = extract_advanced_tof_features(df_seq)\n    features.update(tof_features)\n    \n    # 8. センサー間相互作用特徴量\n    interaction_features = extract_sensor_interaction_features(df_seq)\n    features.update(interaction_features)\n    \n    # 9. 系列レベル特徴量\n    features[\"sequence_length\"] = len(df_seq)\n    \n    # 動作の複雑さ指標\n    features[\"motion_complexity\"] = np.std(linear_acc_mag) + np.std(angular_vel_mag)\n    \n    # 姿勢変化率\n    if len(df_seq) > 1:\n        quat_change = np.diff(df_seq[rot_cols].values, axis=0)\n        features[\"orientation_change_rate\"] = np.mean(np.linalg.norm(quat_change, axis=1))\n    else:\n        features[\"orientation_change_rate\"] = 0\n    \n    return features\n\nprint(\"Advanced feature engineering functions defined\")"},{"cell_type":"markdown","metadata":{},"source":"## Data Loading and Preprocessing"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Load training data\nprint(\"Loading training data...\")\ntrain_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ntrain_demo = pd.read_csv(\n    \"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\"\n)\n\n# Get actual gesture labels from data\nactual_gestures = sorted(train_df[\"gesture\"].unique())\nprint(f\"Actual gesture classes: {actual_gestures}\")\nprint(f\"Data loaded: {len(train_df)} samples\")\nprint(f\"Unique sequences: {train_df['sequence_id'].nunique()}\")\nprint(f\"Number of gesture classes: {len(actual_gestures)}\")\n\n# Check class distribution\nclass_counts = train_df['gesture'].value_counts()\nprint(\"\\nClass distribution:\")\nfor gesture, count in class_counts.items():\n    print(f\"{gesture}: {count} sequences\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Advanced feature extraction with progress tracking\nprint(\"Extracting advanced features...\")\n\nX = []\ny = []\nsequence_ids = train_df[\"sequence_id\"].unique()\n\n# Use tqdm for progress tracking\nfor i, seq_id in enumerate(tqdm(sequence_ids, desc=\"Feature extraction\")):\n    try:\n        # Get sequence data\n        seq_data = train_df[train_df[\"sequence_id\"] == seq_id].copy()\n\n        # Extract advanced physics features\n        features = create_advanced_physics_features(seq_data)\n\n        # Add demographics using subject column\n        subject_id = seq_data[\"subject\"].iloc[0]\n        demo_data = train_demo[train_demo[\"subject\"] == subject_id].iloc[0]\n        features[\"age\"] = demo_data[\"age\"]\n        features[\"is_male\"] = 1 if demo_data[\"sex\"] == \"M\" else 0\n        features[\"height\"] = demo_data.get(\"height_cm\", 170)  # Default height\n\n        # Add subject as categorical feature (for GroupKFold later)\n        features[\"subject_id\"] = subject_id\n\n        X.append(features)\n        y.append(seq_data[\"gesture\"].iloc[0])\n\n        # Memory cleanup every 500 sequences\n        if (i + 1) % 500 == 0:\n            gc.collect()\n\n    except Exception as e:\n        print(f\"Error processing sequence {seq_id}: {e}\")\n        continue\n\n# Convert to DataFrame\nX_df = pd.DataFrame(X)\ny_series = pd.Series(y)\n\nprint(f\"Feature extraction complete: {X_df.shape}\")\nprint(f\"Total features: {len(X_df.columns)}\")\nprint(f\"Sample features: {list(X_df.columns)[:15]}...\")\n\n# Check for NaN values and fill them\nnan_counts = X_df.isnull().sum().sum()\nif nan_counts > 0:\n    print(f\"Found {nan_counts} NaN values, filling with 0\")\n    X_df = X_df.fillna(0)\n\nprint(\"Data preprocessing complete\")"},{"cell_type":"markdown","metadata":{},"source":"## Data Augmentation and Balancing"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Data augmentation for minority classes using SMOTE\ntry:\n    from imblearn.over_sampling import SMOTE, ADASYN\n    from collections import Counter\n    \n    # Prepare data for SMOTE (remove subject_id for SMOTE, add back later)\n    X_for_smote = X_df.drop(['subject_id'], axis=1, errors='ignore')\n    subjects = X_df['subject_id'].values if 'subject_id' in X_df.columns else np.zeros(len(X_df))\n    \n    # Encode labels\n    from sklearn.preprocessing import LabelEncoder\n    le = LabelEncoder()\n    le.fit(actual_gestures)\n    y_encoded = le.transform(y_series)\n    \n    print(\"Original class distribution:\")\n    original_dist = Counter(y_encoded)\n    for class_idx, count in original_dist.items():\n        print(f\"{le.inverse_transform([class_idx])[0]}: {count}\")\n    \n    # Apply SMOTE with conservative parameters\n    smote = SMOTE(\n        sampling_strategy='auto',  # Balance all classes\n        k_neighbors=min(5, min(original_dist.values()) - 1),  # Ensure k < min class size\n        random_state=42\n    )\n    \n    print(\"Applying SMOTE for class balancing...\")\n    X_balanced, y_balanced = smote.fit_resample(X_for_smote, y_encoded)\n    \n    # Create subject IDs for new synthetic samples\n    original_size = len(X_for_smote)\n    new_size = len(X_balanced)\n    \n    # Keep original subjects, assign synthetic subjects with high IDs\n    subjects_balanced = np.concatenate([\n        subjects,\n        np.arange(max(subjects) + 1, max(subjects) + 1 + (new_size - original_size))\n    ])\n    \n    # Add subject_id back to balanced data\n    X_balanced = pd.DataFrame(X_balanced, columns=X_for_smote.columns)\n    X_balanced['subject_id'] = subjects_balanced\n    \n    print(\"Balanced class distribution:\")\n    balanced_dist = Counter(y_balanced)\n    for class_idx, count in balanced_dist.items():\n        print(f\"{le.inverse_transform([class_idx])[0]}: {count}\")\n    \n    print(f\"Data augmented: {X_for_smote.shape} -> {X_balanced.shape}\")\n    \n    # Use balanced data for training\n    X_final = X_balanced\n    y_final = y_balanced\n    \nexcept ImportError:\n    print(\"imbalanced-learn not available, using original data\")\n    X_final = X_df\n    y_final = le.transform(y_series)\nexcept Exception as e:\n    print(f\"SMOTE failed: {e}, using original data\")\n    X_final = X_df\n    y_final = le.transform(y_series)\n\nprint(f\"Final dataset size: {X_final.shape}\")"},{"cell_type":"markdown","metadata":{},"source":"## Advanced Model Training with Optuna Optimization"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import lightgbm as lgb\nimport xgboost as xgb\ntry:\n    import catboost as cb\n    CATBOOST_AVAILABLE = True\nexcept ImportError:\n    CATBOOST_AVAILABLE = False\n    print(\"CatBoost not available, using LightGBM and XGBoost only\")\n\nfrom sklearn.model_selection import cross_val_score, GroupKFold\nfrom sklearn.metrics import accuracy_score, classification_report\n\n# Prepare final training data\nnum_classes = len(le.classes_)\nprint(f\"Number of classes: {num_classes}\")\nprint(f\"Classes: {le.classes_}\")\n\n# Extract subjects for GroupKFold\nsubjects_for_cv = X_final['subject_id'].values\nX_features = X_final.drop(['subject_id'], axis=1)\n\nprint(f\"Features for training: {X_features.shape}\")\nprint(f\"Unique subjects: {len(np.unique(subjects_for_cv))}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Hyperparameter optimization with Optuna\ndef objective_lgb(trial):\n    \"\"\"LightGBM hyperparameter optimization\"\"\"\n    params = {\n        'objective': 'multiclass',\n        'num_class': num_classes,\n        'metric': 'multi_logloss',\n        'boosting_type': 'gbdt',\n        'num_leaves': trial.suggest_int('num_leaves', 50, 300),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n        'feature_fraction': trial.suggest_float('feature_fraction', 0.6, 1.0),\n        'bagging_fraction': trial.suggest_float('bagging_fraction', 0.6, 1.0),\n        'bagging_freq': trial.suggest_int('bagging_freq', 1, 10),\n        'min_child_samples': trial.suggest_int('min_child_samples', 5, 50),\n        'lambda_l1': trial.suggest_float('lambda_l1', 0, 10),\n        'lambda_l2': trial.suggest_float('lambda_l2', 0, 10),\n        'verbose': -1,\n        'random_state': 42,\n        'num_threads': 2\n    }\n    \n    # Cross-validation with GroupKFold\n    gkf = GroupKFold(n_splits=5)\n    scores = []\n    \n    for train_idx, val_idx in gkf.split(X_features, y_final, groups=subjects_for_cv):\n        X_train_fold, X_val_fold = X_features.iloc[train_idx], X_features.iloc[val_idx]\n        y_train_fold, y_val_fold = y_final[train_idx], y_final[val_idx]\n        \n        train_data = lgb.Dataset(X_train_fold, label=y_train_fold)\n        val_data = lgb.Dataset(X_val_fold, label=y_val_fold, reference=train_data)\n        \n        model = lgb.train(\n            params, train_data,\n            num_boost_round=1000,\n            valid_sets=[val_data],\n            callbacks=[lgb.early_stopping(100), lgb.log_evaluation(0)]\n        )\n        \n        pred = model.predict(X_val_fold, num_iteration=model.best_iteration)\n        pred_class = np.argmax(pred, axis=1)\n        score = accuracy_score(y_val_fold, pred_class)\n        scores.append(score)\n    \n    return np.mean(scores)\n\n# Run optimization for LightGBM\nprint(\"Optimizing LightGBM hyperparameters...\")\nstudy_lgb = optuna.create_study(direction='maximize')\nstudy_lgb.optimize(objective_lgb, n_trials=20, timeout=1800)  # 30 minutes max\n\nbest_lgb_params = study_lgb.best_params\nbest_lgb_score = study_lgb.best_value\nprint(f\"Best LightGBM CV score: {best_lgb_score:.4f}\")\nprint(f\"Best LightGBM params: {best_lgb_params}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Hyperparameter optimization for XGBoost\ndef objective_xgb(trial):\n    \"\"\"XGBoost hyperparameter optimization\"\"\"\n    params = {\n        'objective': 'multi:softprob',\n        'num_class': num_classes,\n        'eval_metric': 'mlogloss',\n        'max_depth': trial.suggest_int('max_depth', 3, 10),\n        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n        'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n        'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n        'min_child_weight': trial.suggest_int('min_child_weight', 1, 10),\n        'gamma': trial.suggest_float('gamma', 0, 5),\n        'reg_alpha': trial.suggest_float('reg_alpha', 0, 10),\n        'reg_lambda': trial.suggest_float('reg_lambda', 0, 10),\n        'random_state': 42,\n        'nthread': 2\n    }\n    \n    # Cross-validation with GroupKFold\n    gkf = GroupKFold(n_splits=5)\n    scores = []\n    \n    for train_idx, val_idx in gkf.split(X_features, y_final, groups=subjects_for_cv):\n        X_train_fold, X_val_fold = X_features.iloc[train_idx], X_features.iloc[val_idx]\n        y_train_fold, y_val_fold = y_final[train_idx], y_final[val_idx]\n        \n        dtrain = xgb.DMatrix(X_train_fold, label=y_train_fold)\n        dval = xgb.DMatrix(X_val_fold, label=y_val_fold)\n        \n        model = xgb.train(\n            params, dtrain,\n            num_boost_round=1000,\n            evals=[(dval, 'validation')],\n            early_stopping_rounds=100,\n            verbose_eval=0\n        )\n        \n        pred = model.predict(dval)\n        pred_class = np.argmax(pred, axis=1)\n        score = accuracy_score(y_val_fold, pred_class)\n        scores.append(score)\n    \n    return np.mean(scores)\n\n# Run optimization for XGBoost\nprint(\"Optimizing XGBoost hyperparameters...\")\nstudy_xgb = optuna.create_study(direction='maximize')\nstudy_xgb.optimize(objective_xgb, n_trials=20, timeout=1800)  # 30 minutes max\n\nbest_xgb_params = study_xgb.best_params\nbest_xgb_score = study_xgb.best_value\nprint(f\"Best XGBoost CV score: {best_xgb_score:.4f}\")\nprint(f\"Best XGBoost params: {best_xgb_params}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CatBoost optimization (if available)\nif CATBOOST_AVAILABLE:\n    def objective_cat(trial):\n        \"\"\"CatBoost hyperparameter optimization\"\"\"\n        params = {\n            'objective': 'MultiClass',\n            'eval_metric': 'MultiClass',\n            'iterations': 1000,\n            'depth': trial.suggest_int('depth', 4, 10),\n            'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),\n            'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1, 10),\n            'bootstrap_type': trial.suggest_categorical('bootstrap_type', ['Bayesian', 'Bernoulli']),\n            'random_seed': 42,\n            'verbose': False,\n            'thread_count': 2,\n            'early_stopping_rounds': 100\n        }\n        \n        if params['bootstrap_type'] == 'Bayesian':\n            params['bagging_temperature'] = trial.suggest_float('bagging_temperature', 0, 1)\n        else:\n            params['subsample'] = trial.suggest_float('subsample', 0.6, 1)\n        \n        # Cross-validation with GroupKFold\n        gkf = GroupKFold(n_splits=5)\n        scores = []\n        \n        for train_idx, val_idx in gkf.split(X_features, y_final, groups=subjects_for_cv):\n            X_train_fold, X_val_fold = X_features.iloc[train_idx], X_features.iloc[val_idx]\n            y_train_fold, y_val_fold = y_final[train_idx], y_final[val_idx]\n            \n            model = cb.CatBoostClassifier(**params)\n            model.fit(X_train_fold, y_train_fold, eval_set=(X_val_fold, y_val_fold))\n            \n            pred = model.predict(X_val_fold)\n            score = accuracy_score(y_val_fold, pred)\n            scores.append(score)\n        \n        return np.mean(scores)\n    \n    print(\"Optimizing CatBoost hyperparameters...\")\n    study_cat = optuna.create_study(direction='maximize')\n    study_cat.optimize(objective_cat, n_trials=15, timeout=1800)  # 30 minutes max\n    \n    best_cat_params = study_cat.best_params\n    best_cat_score = study_cat.best_value\n    print(f\"Best CatBoost CV score: {best_cat_score:.4f}\")\n    print(f\"Best CatBoost params: {best_cat_params}\")\nelse:\n    best_cat_params = None\n    best_cat_score = 0"},{"cell_type":"markdown","metadata":{},"source":"## Final Model Training with Optimized Parameters"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"from sklearn.model_selection import train_test_split\n\n# Split data using GroupKFold logic for final training\ngkf = GroupKFold(n_splits=5)\ntrain_idx, val_idx = next(gkf.split(X_features, y_final, groups=subjects_for_cv))\n\nX_train, X_val = X_features.iloc[train_idx], X_features.iloc[val_idx]\ny_train, y_val = y_final[train_idx], y_final[val_idx]\n\nprint(f\"Training set: {X_train.shape}\")\nprint(f\"Validation set: {X_val.shape}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Train optimized LightGBM model\nprint(\"Training optimized LightGBM model...\")\n\nlgb_params = {\n    'objective': 'multiclass',\n    'num_class': num_classes,\n    'metric': 'multi_logloss',\n    'boosting_type': 'gbdt',\n    'verbose': 0,\n    'random_state': 42,\n    'num_threads': 2\n}\nlgb_params.update(best_lgb_params)\n\ntrain_data = lgb.Dataset(X_train, label=y_train)\nvalid_data = lgb.Dataset(X_val, label=y_val)\n\nmodel_lgb = lgb.train(\n    lgb_params,\n    train_data,\n    valid_sets=[valid_data],\n    num_boost_round=2000,\n    callbacks=[lgb.early_stopping(200), lgb.log_evaluation(100)]\n)\n\nprint(\"LightGBM training complete\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Train optimized XGBoost model\nprint(\"Training optimized XGBoost model...\")\n\nxgb_params = {\n    'objective': 'multi:softprob',\n    'num_class': num_classes,\n    'eval_metric': 'mlogloss',\n    'random_state': 42,\n    'nthread': 2\n}\nxgb_params.update(best_xgb_params)\n\ndtrain = xgb.DMatrix(X_train, label=y_train)\ndval = xgb.DMatrix(X_val, label=y_val)\n\nmodel_xgb = xgb.train(\n    xgb_params,\n    dtrain,\n    num_boost_round=2000,\n    evals=[(dval, 'validation')],\n    early_stopping_rounds=200,\n    verbose_eval=100\n)\n\nprint(\"XGBoost training complete\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Train optimized CatBoost model (if available)\nmodel_cat = None\nif CATBOOST_AVAILABLE and best_cat_params:\n    print(\"Training optimized CatBoost model...\")\n    \n    cat_params = {\n        'objective': 'MultiClass',\n        'eval_metric': 'MultiClass',\n        'iterations': 2000,\n        'random_seed': 42,\n        'verbose': 100,\n        'thread_count': 2,\n        'early_stopping_rounds': 200\n    }\n    cat_params.update(best_cat_params)\n    \n    model_cat = cb.CatBoostClassifier(**cat_params)\n    model_cat.fit(X_train, y_train, eval_set=(X_val, y_val))\n    \n    print(\"CatBoost training complete\")\nelse:\n    print(\"CatBoost not available, using LightGBM and XGBoost only\")"},{"cell_type":"markdown","metadata":{},"source":"## Advanced Ensemble and Validation"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Validation performance with advanced ensemble\nprint(\"\\n=== Validation Results ===\")\n\n# LightGBM predictions\npred_lgb = model_lgb.predict(X_val, num_iteration=model_lgb.best_iteration)\npred_lgb_class = np.argmax(pred_lgb, axis=1)\nlgb_accuracy = accuracy_score(y_val, pred_lgb_class)\n\n# XGBoost predictions\ndval = xgb.DMatrix(X_val)\npred_xgb = model_xgb.predict(dval)\npred_xgb_class = np.argmax(pred_xgb, axis=1)\nxgb_accuracy = accuracy_score(y_val, pred_xgb_class)\n\nprint(f\"LightGBM Accuracy: {lgb_accuracy:.4f}\")\nprint(f\"XGBoost Accuracy: {xgb_accuracy:.4f}\")\n\n# CatBoost predictions (if available)\nif model_cat is not None:\n    pred_cat = model_cat.predict_proba(X_val)\n    pred_cat_class = np.argmax(pred_cat, axis=1)\n    cat_accuracy = accuracy_score(y_val, pred_cat_class)\n    print(f\"CatBoost Accuracy: {cat_accuracy:.4f}\")\n    \n    # Three-model ensemble\n    ensemble_weights = [0.4, 0.35, 0.25]  # LightGBM, XGBoost, CatBoost\n    pred_ensemble = (ensemble_weights[0] * pred_lgb + \n                    ensemble_weights[1] * pred_xgb + \n                    ensemble_weights[2] * pred_cat)\nelse:\n    # Two-model ensemble\n    ensemble_weights = [0.55, 0.45]  # LightGBM, XGBoost\n    pred_ensemble = ensemble_weights[0] * pred_lgb + ensemble_weights[1] * pred_xgb\n\npred_ensemble_class = np.argmax(pred_ensemble, axis=1)\nensemble_accuracy = accuracy_score(y_val, pred_ensemble_class)\n\nprint(f\"\\n🎯 Ensemble Accuracy: {ensemble_accuracy:.4f}\")\nprint(f\"Target: 87%+ ({'✅ ACHIEVED' if ensemble_accuracy >= 0.87 else '❌ NOT YET'})\")\n\n# Detailed classification report\nprint(\"\\n=== Detailed Classification Report ===\")\ntarget_names = [str(cls) for cls in le.classes_]\nprint(classification_report(y_val, pred_ensemble_class, target_names=target_names))"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Save all models\nprint(\"Saving optimized models...\")\n\n# Save models\nwith open(\"advanced_model_lgb_v5.pkl\", \"wb\") as f:\n    pickle.dump(model_lgb, f)\n\nwith open(\"advanced_model_xgb_v5.pkl\", \"wb\") as f:\n    pickle.dump(model_xgb, f)\n\nif model_cat is not None:\n    model_cat.save_model(\"advanced_model_cat_v5.cbm\")\n\n# Save label encoder and parameters\nwith open(\"advanced_label_encoder_v5.pkl\", \"wb\") as f:\n    pickle.dump(le, f)\n\n# Save ensemble weights and model info\nmodel_info = {\n    'ensemble_weights': ensemble_weights,\n    'has_catboost': model_cat is not None,\n    'lgb_params': lgb_params,\n    'xgb_params': xgb_params,\n    'validation_accuracy': ensemble_accuracy,\n    'feature_names': list(X_features.columns)\n}\n\nif model_cat is not None:\n    model_info['cat_params'] = cat_params\n\nwith open(\"advanced_model_info_v5.pkl\", \"wb\") as f:\n    pickle.dump(model_info, f)\n\nprint(\"Models saved successfully\")\nprint(f\"Final validation accuracy: {ensemble_accuracy:.4f}\")"},{"cell_type":"markdown","metadata":{},"source":"## Advanced Inference Function"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def predict(sequence, demographics):\n    \"\"\"\n    Advanced physics-based prediction with optimized ensemble model.\n    Target: 87%+ validation accuracy with comprehensive feature engineering.\n    \"\"\"\n    try:\n        # Convert to DataFrame\n        df_seq = pd.DataFrame(sequence)\n\n        # Extract comprehensive advanced features\n        features = create_advanced_physics_features(df_seq)\n\n        # Add demographics (using lowercase keys from CMI server)\n        features[\"age\"] = demographics.get(\"age\", 30)\n        features[\"is_male\"] = 1 if demographics.get(\"sex\", \"M\") == \"M\" else 0\n        features[\"height\"] = demographics.get(\"height_cm\", 170)\n        \n        # Add dummy subject_id for inference (not used in prediction)\n        features[\"subject_id\"] = 0\n\n        # Convert to DataFrame and ensure correct feature order\n        X = pd.DataFrame([features])\n        X_features_pred = X[model_info['feature_names']]  # Ensure correct feature order\n\n        # LightGBM prediction\n        pred_lgb = model_lgb.predict(X_features_pred, num_iteration=model_lgb.best_iteration)\n\n        # XGBoost prediction\n        dtest = xgb.DMatrix(X_features_pred)\n        pred_xgb = model_xgb.predict(dtest)\n\n        # CatBoost prediction (if available)\n        if model_info['has_catboost'] and model_cat is not None:\n            pred_cat = model_cat.predict_proba(X_features_pred)\n            \n            # Three-model ensemble\n            pred_ensemble = (model_info['ensemble_weights'][0] * pred_lgb + \n                           model_info['ensemble_weights'][1] * pred_xgb + \n                           model_info['ensemble_weights'][2] * pred_cat)\n        else:\n            # Two-model ensemble\n            pred_ensemble = (model_info['ensemble_weights'][0] * pred_lgb + \n                           model_info['ensemble_weights'][1] * pred_xgb)\n\n        # Get predicted class\n        pred_class_idx = np.argmax(pred_ensemble[0])\n        pred_gesture = le.inverse_transform([pred_class_idx])[0]\n\n        # Memory cleanup\n        del df_seq, features, X, X_features_pred, pred_lgb, pred_xgb, pred_ensemble\n        if model_info['has_catboost']:\n            del pred_cat\n        gc.collect()\n\n        return pred_gesture\n\n    except Exception as e:\n        print(f\"Advanced prediction error: {e}\")\n        # Fallback to most common gesture from training data\n        return \"Text on phone\"  # Most common in training data\n\n\nprint(\"Advanced prediction function defined\")\nprint(f\"Expected validation accuracy: {ensemble_accuracy:.4f}\")\nprint(f\"Target achievement: {'✅ SUCCESS' if ensemble_accuracy >= 0.87 else '❌ NEEDS MORE WORK'}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Test the advanced prediction function\nprint(\"Testing advanced prediction function...\")\n\n# Get a sample sequence from validation set\ntest_seq_id = train_df[\"sequence_id\"].unique()[val_idx[0]]  # Use validation sequence\ntest_seq = train_df[train_df[\"sequence_id\"] == test_seq_id].drop(\n    [\n        \"sequence_id\", \"behavior\", \"gesture\", \"subject\", \"orientation\", \n        \"phase\", \"row_id\", \"sequence_type\", \"sequence_counter\"\n    ],\n    axis=1, errors='ignore'\n)\n\n# Get demographics for test\ntest_subject = train_df[train_df[\"sequence_id\"] == test_seq_id][\"subject\"].iloc[0]\ntest_demo_row = train_demo[train_demo[\"subject\"] == test_subject].iloc[0]\ntest_demo = {\n    \"age\": test_demo_row[\"age\"],\n    \"sex\": test_demo_row[\"sex\"],\n    \"height_cm\": test_demo_row.get(\"height_cm\", 170),\n}\n\n# Test prediction\ntest_pred = predict(test_seq.to_dict(\"records\"), test_demo)\nactual_gesture = train_df[train_df[\"sequence_id\"] == test_seq_id][\"gesture\"].iloc[0]\n\nprint(f\"Test prediction: {test_pred}\")\nprint(f\"Actual gesture: {actual_gesture}\")\nprint(f\"Match: {test_pred == actual_gesture}\")\nprint(f\"\\nAdvanced model ready with {ensemble_accuracy:.4f} validation accuracy\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Start inference server\nprint(\"Starting Advanced CMI Inference Server...\")\nprint(f\"Model validation accuracy: {ensemble_accuracy:.4f}\")\nprint(f\"Target 87%+: {'✅ ACHIEVED' if ensemble_accuracy >= 0.87 else '❌ WORKING ON IT'}\")\nprint(\"Server will handle predictions for test data with advanced feature engineering\")\n\nserver = CMIInferenceServer(predict)\nserver.serve()\n\nprint(\"Advanced inference complete\")"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.5"}},"nbformat":4,"nbformat_minor":4}