{"cells":[{"cell_type":"markdown","metadata":{},"source":"# CMI Hybrid Ensemble V12 - 合成データ + 実データモデル\n\nV11（合成データ）とV12（実データ）モデルのハイブリッドアンサンブル推論"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport pickle\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# 環境変数を確認\nIS_KAGGLE = os.getenv('KAGGLE_IS_COMPETITION_RERUN') is not None\nprint(f\"Running on Kaggle: {IS_KAGGLE}\")\n\nif IS_KAGGLE:\n    DATA_PATH = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test'\n    # V11モデル（合成データモデル）\n    V11_MODEL_PATH = '/kaggle/input/cmi-v11-dataset'\n    # V12モデル（実データモデル）\n    V12_MODEL_PATH = '/kaggle/input/cmi-v12-real-data-models'\nelse:\n    DATA_PATH = '/kaggle/input/cmi-detect-behavior-with-sensor-data/test'\n    V11_MODEL_PATH = 'cmi-v11-dataset'\n    V12_MODEL_PATH = 'models_v12'"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# モデルをロード\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom catboost import CatBoostClassifier\nfrom sklearn.preprocessing import StandardScaler\nimport joblib\n\nprint(\"Loading V11 models (synthetic data)...\")\n# V11モデル - 最終モデルのみ使用\nlgb_model_v11 = lgb.Booster(model_file=os.path.join(V11_MODEL_PATH, 'lgb_model_v11.txt'))\ncat_model_v11 = CatBoostClassifier()\ncat_model_v11.load_model(os.path.join(V11_MODEL_PATH, 'cat_model_v11.cbm'))\n\nprint(\"Loading V12 models (real data)...\")\n# V12モデル - 最終モデルのみ使用\nlgb_model_v12 = lgb.Booster(model_file=os.path.join(V12_MODEL_PATH, 'lgb_model_v12_final.txt'))\nxgb_model_v12 = xgb.Booster()\nxgb_model_v12.load_model(os.path.join(V12_MODEL_PATH, 'xgb_model_v12_final.json'))\ncat_model_v12 = CatBoostClassifier()\ncat_model_v12.load_model(os.path.join(V12_MODEL_PATH, 'cat_model_v12_final.cbm'))\n\n# V12用のscalerとその他のメタデータ\nwith open(os.path.join(V12_MODEL_PATH, 'scaler_v12.pkl'), 'rb') as f:\n    scaler_v12 = pickle.load(f)\nwith open(os.path.join(V12_MODEL_PATH, 'feature_names_v12.pkl'), 'rb') as f:\n    feature_names_v12 = pickle.load(f)\nwith open(os.path.join(V12_MODEL_PATH, 'gesture_mapping_v12.pkl'), 'rb') as f:\n    gesture_mapping_v12 = pickle.load(f)\n\nprint(f\"Loaded models successfully\")\nprint(f\"V12 features: {len(feature_names_v12)}\")\nprint(f\"Gesture mapping: {gesture_mapping_v12}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def extract_advanced_features(df, seq_id, metadata_row):\n    \"\"\"V12で使用する高度な特徴量抽出（V12と同じ実装）\"\"\"\n    features = {}\n    \n    # 基本統計量\n    for col in df.columns:\n        if col != 'timestamp':\n            features[f'{col}_mean'] = np.mean(df[col])\n            features[f'{col}_std'] = np.std(df[col])\n            features[f'{col}_min'] = np.min(df[col])\n            features[f'{col}_max'] = np.max(df[col])\n            features[f'{col}_median'] = np.median(df[col])\n            features[f'{col}_q25'] = np.percentile(df[col], 25)\n            features[f'{col}_q75'] = np.percentile(df[col], 75)\n            features[f'{col}_iqr'] = features[f'{col}_q75'] - features[f'{col}_q25']\n            features[f'{col}_skew'] = df[col].skew() if len(df[col]) > 2 else 0\n            features[f'{col}_kurtosis'] = df[col].kurtosis() if len(df[col]) > 3 else 0\n    \n    # 重力除去線形加速度\n    gravity_x = np.mean(df['acc_x'])\n    gravity_y = np.mean(df['acc_y'])\n    gravity_z = np.mean(df['acc_z'])\n    \n    linear_acc_x = df['acc_x'] - gravity_x\n    linear_acc_y = df['acc_y'] - gravity_y\n    linear_acc_z = df['acc_z'] - gravity_z\n    \n    features['linear_acc_x_mean'] = np.mean(linear_acc_x)\n    features['linear_acc_y_mean'] = np.mean(linear_acc_y)\n    features['linear_acc_z_mean'] = np.mean(linear_acc_z)\n    features['linear_acc_magnitude'] = np.mean(np.sqrt(linear_acc_x**2 + linear_acc_y**2 + linear_acc_z**2))\n    \n    # クォータニオンからオイラー角\n    qw = df['quat_w'].values\n    qx = df['quat_x'].values\n    qy = df['quat_y'].values\n    qz = df['quat_z'].values\n    \n    # ロール、ピッチ、ヨー\n    roll = np.arctan2(2*(qw*qx + qy*qz), 1 - 2*(qx**2 + qy**2))\n    pitch = np.arcsin(np.clip(2*(qw*qy - qz*qx), -1, 1))\n    yaw = np.arctan2(2*(qw*qz + qx*qy), 1 - 2*(qy**2 + qz**2))\n    \n    features['euler_roll_mean'] = np.mean(roll)\n    features['euler_pitch_mean'] = np.mean(pitch)\n    features['euler_yaw_mean'] = np.mean(yaw)\n    features['euler_roll_std'] = np.std(roll)\n    features['euler_pitch_std'] = np.std(pitch)\n    features['euler_yaw_std'] = np.std(yaw)\n    \n    # 角速度（クォータニオンの時間微分）\n    if len(df) > 1:\n        dt = np.diff(df['timestamp'].values) / 1000.0\n        dqw = np.diff(qw) / dt\n        dqx = np.diff(qx) / dt\n        dqy = np.diff(qy) / dt\n        dqz = np.diff(qz) / dt\n        \n        features['angular_velocity_mean'] = np.mean(np.sqrt(dqx**2 + dqy**2 + dqz**2))\n        features['angular_velocity_std'] = np.std(np.sqrt(dqx**2 + dqy**2 + dqz**2))\n    else:\n        features['angular_velocity_mean'] = 0\n        features['angular_velocity_std'] = 0\n    \n    # ジャーク（加速度の変化率）\n    if len(df) > 1:\n        jerk_x = np.diff(df['acc_x']) / dt\n        jerk_y = np.diff(df['acc_y']) / dt\n        jerk_z = np.diff(df['acc_z']) / dt\n        \n        features['jerk_x_mean'] = np.mean(jerk_x)\n        features['jerk_y_mean'] = np.mean(jerk_y)\n        features['jerk_z_mean'] = np.mean(jerk_z)\n        features['jerk_magnitude'] = np.mean(np.sqrt(jerk_x**2 + jerk_y**2 + jerk_z**2))\n    else:\n        features['jerk_x_mean'] = 0\n        features['jerk_y_mean'] = 0\n        features['jerk_z_mean'] = 0\n        features['jerk_magnitude'] = 0\n    \n    # ToFセンサー統計\n    tof_cols = [col for col in df.columns if col.startswith('tof_')]\n    if tof_cols:\n        tof_values = df[tof_cols].values.flatten()\n        features['tof_global_mean'] = np.mean(tof_values)\n        features['tof_global_std'] = np.std(tof_values)\n        features['tof_global_min'] = np.min(tof_values)\n        features['tof_global_max'] = np.max(tof_values)\n    \n    # シーケンス長\n    features['sequence_length'] = len(df)\n    features['duration_ms'] = df['timestamp'].max() - df['timestamp'].min()\n    \n    return features\n\ndef extract_simple_features(df, seq_id, metadata_row):\n    \"\"\"V11で使用するシンプルな特徴量抽出\"\"\"\n    features = {}\n    \n    # IMUセンサーデータの基本統計量\n    imu_cols = ['acc_x', 'acc_y', 'acc_z', 'quat_w', 'quat_x', 'quat_y', 'quat_z']\n    for col in imu_cols:\n        if col in df.columns:\n            features[f'{col}_mean'] = df[col].mean()\n            features[f'{col}_std'] = df[col].std()\n            features[f'{col}_min'] = df[col].min()\n            features[f'{col}_max'] = df[col].max()\n    \n    # ToFセンサーの統計量\n    tof_cols = [col for col in df.columns if col.startswith('tof_')]\n    if tof_cols:\n        tof_values = df[tof_cols].values.flatten()\n        features['tof_mean'] = np.mean(tof_values)\n        features['tof_std'] = np.std(tof_values)\n        features['tof_min'] = np.min(tof_values)\n        features['tof_max'] = np.max(tof_values)\n    \n    # 加速度の大きさ\n    if all(col in df.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n        acc_magnitude = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n        features['acc_magnitude_mean'] = acc_magnitude.mean()\n        features['acc_magnitude_std'] = acc_magnitude.std()\n    \n    # シーケンス長\n    features['sequence_length'] = len(df)\n    \n    return features"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# CMI推論サーバーのセットアップ\nif IS_KAGGLE:\n    from kaggle_evaluation.cmi_inference_server import CMIInferenceServer\n    \n    def predict(test_data: pl.DataFrame, metadata: pl.DataFrame) -> pl.DataFrame:\n        \"\"\"ハイブリッドアンサンブル予測\"\"\"\n        print(f\"Processing {len(metadata)} sequences for prediction\")\n        \n        # pandas DataFrameに変換\n        test_df = test_data.to_pandas()\n        metadata_df = metadata.to_pandas()\n        \n        predictions = []\n        \n        for idx, row in metadata_df.iterrows():\n            seq_id = row['sequence_id']\n            \n            # シーケンスデータを取得\n            seq_data = test_df[test_df['sequence_id'] == seq_id].copy()\n            \n            if len(seq_data) == 0:\n                print(f\"Warning: No data for sequence {seq_id}\")\n                predictions.append(0)  # デフォルト予測\n                continue\n            \n            # V11用の特徴量抽出（シンプル）\n            features_v11 = extract_simple_features(seq_data, seq_id, row)\n            X_v11 = pd.DataFrame([features_v11])\n            \n            # V11予測（LightGBM + CatBoost）\n            pred_lgb_v11 = lgb_model_v11.predict(X_v11, num_iteration=lgb_model_v11.best_iteration)[0]\n            pred_cat_v11 = cat_model_v11.predict_proba(X_v11)[0]\n            \n            # V11アンサンブル（LightGBM 50% + CatBoost 50%）\n            pred_v11 = 0.5 * pred_lgb_v11 + 0.5 * pred_cat_v11\n            \n            # V12用の特徴量抽出（高度）\n            features_v12 = extract_advanced_features(seq_data, seq_id, row)\n            X_v12 = pd.DataFrame([features_v12])\n            \n            # V12用の特徴量を整形\n            X_v12_aligned = X_v12[feature_names_v12]\n            X_v12_scaled = scaler_v12.transform(X_v12_aligned)\n            \n            # V12予測（LightGBM + XGBoost + CatBoost）\n            pred_lgb_v12 = lgb_model_v12.predict(X_v12_scaled, num_iteration=lgb_model_v12.best_iteration)[0]\n            \n            dmatrix_v12 = xgb.DMatrix(X_v12_scaled, feature_names=feature_names_v12)\n            pred_xgb_v12 = xgb_model_v12.predict(dmatrix_v12)[0]\n            \n            pred_cat_v12 = cat_model_v12.predict_proba(X_v12_scaled)[0]\n            \n            # V12アンサンブル（LightGBM 40% + XGBoost 30% + CatBoost 30%）\n            pred_v12 = 0.4 * pred_lgb_v12 + 0.3 * pred_xgb_v12 + 0.3 * pred_cat_v12\n            \n            # ハイブリッドアンサンブル\n            # V11（合成データ）20% + V12（実データ）80%\n            hybrid_probs = 0.2 * pred_v11 + 0.8 * pred_v12\n            \n            # 最も確率の高いクラスを選択\n            pred_class = np.argmax(hybrid_probs)\n            \n            # gesture_mappingで元のラベルに戻す\n            reverse_mapping = {v: k for k, v in gesture_mapping_v12.items()}\n            original_label = reverse_mapping.get(pred_class, pred_class)\n            \n            predictions.append(original_label)\n        \n        # Polars DataFrameとして返す\n        result = pl.DataFrame({\n            'sequence_id': metadata['sequence_id'],\n            'gesture': predictions\n        })\n        \n        print(f\"Predictions completed. Shape: {result.shape}\")\n        print(f\"Unique gestures predicted: {result['gesture'].unique().to_list()}\")\n        \n        return result\n    \n    # 推論サーバーを開始\n    print(\"Starting CMI Inference Server with hybrid ensemble...\")\n    inference_server = CMIInferenceServer(predict)\n    inference_server.serve()\n    print(\"Inference server completed\")\n    \nelse:\n    print(\"Local testing mode - Skipping inference server\")\n    print(\"Models loaded successfully for hybrid ensemble:\")\n    print(\"- V11: LightGBM + CatBoost (synthetic data, 20% weight)\")\n    print(\"- V12: LightGBM + XGBoost + CatBoost (real data, 80% weight)\")"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}