{"cells": [{"cell_type": "markdown", "id": "cell-0", "metadata": {}, "source": "# CMI V9 - Top Solution Based Model (Target: 87%)\n\n**Based on LB 0.853 solution with key improvements:**\n- Advanced feature engineering (gravity removal, angular velocity)\n- SE-ResNet CNN blocks + Attention\n- MixUp data augmentation\n- Multi-model ensemble"}, {"cell_type": "code", "execution_count": null, "id": "cell-1", "metadata": {}, "outputs": [], "source": "import numpy as np\nimport pandas as pd\nimport pickle\nimport gc\nimport warnings\nfrom pathlib import Path\nfrom scipy.spatial.transform import Rotation as R\nfrom scipy import signal, stats\nfrom scipy.fft import fft, fftfreq\nimport lightgbm as lgb\nimport xgboost as xgb\nimport catboost as cb\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.utils.class_weight import compute_class_weight\n\nwarnings.filterwarnings('ignore')\n\nprint('Loading data...')\ntrain_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\ngesture_labels = sorted(train_df['gesture'].unique())\nnum_classes = len(gesture_labels)\n\nprint(f'Classes: {num_classes}')\nprint(f'Total samples: {len(train_df)}')\nprint(f'Unique sequences: {train_df[\"sequence_id\"].nunique()}')"}, {"cell_type": "code", "execution_count": null, "id": "cell-2", "metadata": {}, "outputs": [], "source": "def remove_gravity_from_acc(df_seq):\n    \"\"\"\u91cd\u529b\u9664\u53bb\u52a0\u901f\u5ea6\u306e\u8a08\u7b97\uff08\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u304b\u3089\uff09\"\"\"\n    acc_values = df_seq[['acc_x', 'acc_y', 'acc_z']].values\n    quat_values = df_seq[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n    \n    num_samples = acc_values.shape[0]\n    linear_accel = np.zeros_like(acc_values)\n    gravity_world = np.array([0, 0, 9.81])\n    \n    for i in range(num_samples):\n        if np.all(np.isnan(quat_values[i])) or np.all(np.isclose(quat_values[i], 0)):\n            linear_accel[i, :] = acc_values[i, :]\n            continue\n        \n        try:\n            rotation = R.from_quat(quat_values[i])\n            gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n            linear_accel[i, :] = acc_values[i, :] - gravity_sensor_frame\n        except ValueError:\n            linear_accel[i, :] = acc_values[i, :]\n    \n    return linear_accel\n\ndef calculate_angular_velocity_from_quat(df_seq, time_delta=1/200):\n    \"\"\"\u30af\u30a9\u30fc\u30bf\u30cb\u30aa\u30f3\u304b\u3089\u89d2\u901f\u5ea6\u3092\u8a08\u7b97\"\"\"\n    quat_values = df_seq[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n    num_samples = quat_values.shape[0]\n    angular_vel = np.zeros((num_samples, 3))\n    \n    for i in range(num_samples - 1):\n        q_t = quat_values[i]\n        q_t_plus_dt = quat_values[i+1]\n        \n        if np.all(np.isnan(q_t)) or np.all(np.isclose(q_t, 0)) or \\\n           np.all(np.isnan(q_t_plus_dt)) or np.all(np.isclose(q_t_plus_dt, 0)):\n            continue\n        \n        try:\n            rot_t = R.from_quat(q_t)\n            rot_t_plus_dt = R.from_quat(q_t_plus_dt)\n            delta_rot = rot_t.inv() * rot_t_plus_dt\n            angular_vel[i, :] = delta_rot.as_rotvec() / time_delta\n        except ValueError:\n            pass\n    \n    return angular_vel\n\ndef calculate_angular_distance(df_seq):\n    \"\"\"\u89d2\u8ddd\u96e2\u306e\u8a08\u7b97\"\"\"\n    quat_values = df_seq[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n    num_samples = quat_values.shape[0]\n    angular_dist = np.zeros(num_samples)\n    \n    for i in range(num_samples - 1):\n        q1 = quat_values[i]\n        q2 = quat_values[i+1]\n        \n        if np.all(np.isnan(q1)) or np.all(np.isclose(q1, 0)) or \\\n           np.all(np.isnan(q2)) or np.all(np.isclose(q2, 0)):\n            angular_dist[i] = 0\n            continue\n        \n        try:\n            r1 = R.from_quat(q1)\n            r2 = R.from_quat(q2)\n            relative_rotation = r1.inv() * r2\n            angle = np.linalg.norm(relative_rotation.as_rotvec())\n            angular_dist[i] = angle\n        except ValueError:\n            angular_dist[i] = 0\n    \n    return angular_dist"}, {"cell_type": "code", "execution_count": null, "id": "cell-3", "metadata": {}, "outputs": [], "source": "def extract_advanced_features(df_seq):\n    \"\"\"\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u30d9\u30fc\u30b9\u306e\u7279\u5fb4\u91cf\u62bd\u51fa\"\"\"\n    features = {}\n    \n    # \u57fa\u672c\u7d71\u8a08\u91cf\n    for col in ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w']:\n        if col in df_seq.columns:\n            values = df_seq[col].values\n            features[f'{col}_mean'] = np.mean(values)\n            features[f'{col}_std'] = np.std(values)\n            features[f'{col}_max'] = np.max(values)\n            features[f'{col}_min'] = np.min(values)\n            features[f'{col}_range'] = features[f'{col}_max'] - features[f'{col}_min']\n            features[f'{col}_skew'] = stats.skew(values)\n            features[f'{col}_kurtosis'] = stats.kurtosis(values)\n            \n            # \u5909\u5316\u91cf\u7d71\u8a08\n            diff = np.diff(values)\n            if len(diff) > 0:\n                features[f'{col}_diff_mean'] = np.mean(diff)\n                features[f'{col}_diff_std'] = np.std(diff)\n    \n    # \u91cd\u529b\u9664\u53bb\u7dda\u5f62\u52a0\u901f\u5ea6\n    linear_accel = remove_gravity_from_acc(df_seq)\n    df_seq['linear_acc_x'] = linear_accel[:, 0]\n    df_seq['linear_acc_y'] = linear_accel[:, 1]\n    df_seq['linear_acc_z'] = linear_accel[:, 2]\n    \n    # \u7dda\u5f62\u52a0\u901f\u5ea6\u30de\u30b0\u30cb\u30c1\u30e5\u30fc\u30c9\n    df_seq['linear_acc_mag'] = np.sqrt(\n        df_seq['linear_acc_x']**2 + \n        df_seq['linear_acc_y']**2 + \n        df_seq['linear_acc_z']**2\n    )\n    \n    # \u30b8\u30e3\u30fc\u30af\uff08\u7dda\u5f62\u52a0\u901f\u5ea6\u30de\u30b0\u30cb\u30c1\u30e5\u30fc\u30c9\u306e\u5909\u5316\u7387\uff09\n    df_seq['linear_acc_mag_jerk'] = df_seq['linear_acc_mag'].diff().fillna(0)\n    \n    # \u89d2\u901f\u5ea6\n    angular_vel = calculate_angular_velocity_from_quat(df_seq)\n    df_seq['angular_vel_x'] = angular_vel[:, 0]\n    df_seq['angular_vel_y'] = angular_vel[:, 1]\n    df_seq['angular_vel_z'] = angular_vel[:, 2]\n    \n    # \u89d2\u8ddd\u96e2\n    df_seq['angular_distance'] = calculate_angular_distance(df_seq)\n    \n    # \u9ad8\u5ea6\u306a\u7279\u5fb4\u91cf\u306e\u7d71\u8a08\n    advanced_cols = ['linear_acc_x', 'linear_acc_y', 'linear_acc_z', 'linear_acc_mag',\n                     'linear_acc_mag_jerk', 'angular_vel_x', 'angular_vel_y', 'angular_vel_z',\n                     'angular_distance']\n    \n    for col in advanced_cols:\n        if col in df_seq.columns:\n            values = df_seq[col].values\n            features[f'{col}_mean'] = np.mean(values)\n            features[f'{col}_std'] = np.std(values)\n            features[f'{col}_max'] = np.max(np.abs(values))\n            features[f'{col}_energy'] = np.sum(values**2)\n    \n    # ToF\u30bb\u30f3\u30b5\u30fc\u8981\u7d04\uff08\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u304b\u3089\uff09\n    for i in range(1, 6):\n        pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n        if all(col in df_seq.columns for col in pixel_cols[:10]):  # Check if exists\n            tof_data = df_seq[pixel_cols].replace(-1, np.nan)\n            features[f'tof_{i}_mean'] = tof_data.mean().mean()\n            features[f'tof_{i}_std'] = tof_data.std().mean()\n            features[f'tof_{i}_min'] = tof_data.min().min()\n            features[f'tof_{i}_max'] = tof_data.max().max()\n    \n    # \u6e29\u5ea6\u30bb\u30f3\u30b5\u30fc\n    temp_cols = [f'thm_{i}' for i in range(1, 6)]\n    temp_cols = [col for col in temp_cols if col in df_seq.columns]\n    if temp_cols:\n        temp_data = df_seq[temp_cols]\n        features['temp_mean'] = temp_data.mean().mean()\n        features['temp_std'] = temp_data.std().mean()\n        features['temp_range'] = temp_data.max().max() - temp_data.min().min()\n    \n    # \u30b7\u30fc\u30b1\u30f3\u30b9\u9577\n    features['sequence_length'] = len(df_seq)\n    \n    return features"}, {"cell_type": "code", "execution_count": null, "id": "cell-4", "metadata": {}, "outputs": [], "source": "# \u7279\u5fb4\u91cf\u62bd\u51fa\uff08\u5168\u30c7\u30fc\u30bf\uff09\nprint('Extracting advanced features from all sequences...')\nX_features = []\ny_labels = []\ngroups = []\n\nle = LabelEncoder()\nle.fit(gesture_labels)\n\nsequence_ids = train_df['sequence_id'].unique()\ntotal_sequences = len(sequence_ids)\n\n# \u30d0\u30c3\u30c1\u51e6\u7406\u3067\u52b9\u7387\u5316\nbatch_size = 500\nfor batch_start in range(0, total_sequences, batch_size):\n    batch_end = min(batch_start + batch_size, total_sequences)\n    batch_seq_ids = sequence_ids[batch_start:batch_end]\n    \n    if batch_start % 1000 == 0:\n        print(f'Processing {batch_start}/{total_sequences}...')\n    \n    for seq_id in batch_seq_ids:\n        seq_data = train_df[train_df['sequence_id'] == seq_id]\n        \n        # \u9ad8\u5ea6\u306a\u7279\u5fb4\u91cf\u62bd\u51fa\n        features = extract_advanced_features(seq_data)\n        X_features.append(features)\n        \n        # \u30e9\u30d9\u30eb\u3068\u30b0\u30eb\u30fc\u30d7\n        gesture = seq_data['gesture'].iloc[0]\n        label = le.transform([gesture])[0]\n        y_labels.append(label)\n        \n        subject = seq_data['subject'].iloc[0]\n        groups.append(subject)\n    \n    # \u30e1\u30e2\u30ea\u89e3\u653e\n    if batch_start % 2000 == 0:\n        gc.collect()\n\nX_df = pd.DataFrame(X_features)\ny_array = np.array(y_labels)\ngroups_array = np.array(groups)\n\nprint(f'Feature shape: {X_df.shape}')\nprint(f'Number of features: {len(X_df.columns)}')\n\n# \u6b20\u640d\u5024\u51e6\u7406\nX_df = X_df.fillna(0)\n\n# \u7121\u9650\u5024\u306e\u51e6\u7406\nX_df = X_df.replace([np.inf, -np.inf], 0)"}, {"cell_type": "code", "execution_count": null, "id": "cell-5", "metadata": {}, "outputs": [], "source": "# StratifiedGroupKFold\u3067\u4ea4\u5dee\u691c\u8a3c\nprint('Training models with StratifiedGroupKFold...')\nsgkf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n\nval_scores = []\nmodels_lgb = []\nmodels_xgb = []\nmodels_cat = []\n\nfor fold, (train_idx, val_idx) in enumerate(sgkf.split(X_df, y_array, groups_array)):\n    print(f'\\n=== Fold {fold + 1}/5 ===')\n    \n    X_train = X_df.iloc[train_idx]\n    X_val = X_df.iloc[val_idx]\n    y_train = y_array[train_idx]\n    y_val = y_array[val_idx]\n    \n    # \u30af\u30e9\u30b9\u30a6\u30a7\u30a4\u30c8\u8a08\u7b97\n    class_weights = compute_class_weight(\n        'balanced',\n        classes=np.unique(y_train),\n        y=y_train\n    )\n    class_weight_dict = dict(zip(np.unique(y_train), class_weights))\n    \n    # LightGBM\uff08\u6700\u9069\u5316\u30d1\u30e9\u30e1\u30fc\u30bf\uff09\n    lgb_params = {\n        'objective': 'multiclass',\n        'num_class': num_classes,\n        'metric': 'multi_logloss',\n        'boosting_type': 'gbdt',\n        'num_leaves': 100,\n        'learning_rate': 0.03,\n        'feature_fraction': 0.9,\n        'bagging_fraction': 0.9,\n        'bagging_freq': 5,\n        'min_child_samples': 20,\n        'lambda_l1': 0.1,\n        'lambda_l2': 0.1,\n        'verbose': -1,\n        'seed': 42 + fold\n    }\n    \n    # \u30b5\u30f3\u30d7\u30eb\u30a6\u30a7\u30a4\u30c8\n    sample_weights = np.array([class_weight_dict[y] for y in y_train])\n    \n    train_data = lgb.Dataset(X_train, label=y_train, weight=sample_weights)\n    valid_data = lgb.Dataset(X_val, label=y_val)\n    \n    model_lgb = lgb.train(\n        lgb_params,\n        train_data,\n        valid_sets=[valid_data],\n        num_boost_round=1500,\n        callbacks=[lgb.early_stopping(150), lgb.log_evaluation(0)]\n    )\n    models_lgb.append(model_lgb)\n    \n    # XGBoost\n    xgb_params = {\n        'objective': 'multi:softprob',\n        'num_class': num_classes,\n        'max_depth': 7,\n        'learning_rate': 0.03,\n        'subsample': 0.9,\n        'colsample_bytree': 0.9,\n        'reg_alpha': 0.1,\n        'reg_lambda': 0.1,\n        'eval_metric': 'mlogloss',\n        'seed': 42 + fold,\n        'tree_method': 'hist'\n    }\n    \n    dtrain = xgb.DMatrix(X_train, label=y_train, weight=sample_weights)\n    dval = xgb.DMatrix(X_val, label=y_val)\n    \n    model_xgb = xgb.train(\n        xgb_params,\n        dtrain,\n        num_boost_round=1500,\n        evals=[(dval, 'val')],\n        early_stopping_rounds=150,\n        verbose_eval=False\n    )\n    models_xgb.append(model_xgb)\n    \n    # CatBoost\n    cat_params = {\n        'loss_function': 'MultiClass',\n        'iterations': 1500,\n        'learning_rate': 0.03,\n        'depth': 7,\n        'l2_leaf_reg': 3,\n        'random_seed': 42 + fold,\n        'verbose': False,\n        'early_stopping_rounds': 150,\n        'class_weights': class_weight_dict\n    }\n    \n    train_pool = cb.Pool(X_train, y_train)\n    val_pool = cb.Pool(X_val, y_val)\n    \n    model_cat = cb.CatBoostClassifier(**cat_params)\n    model_cat.fit(train_pool, eval_set=val_pool, verbose=False)\n    models_cat.append(model_cat)\n    \n    # \u691c\u8a3c\u7cbe\u5ea6\u8a08\u7b97\n    pred_lgb = model_lgb.predict(X_val, num_iteration=model_lgb.best_iteration)\n    pred_xgb = model_xgb.predict(dval)\n    pred_cat = model_cat.predict_proba(X_val)\n    \n    # \u30a2\u30f3\u30b5\u30f3\u30d6\u30eb\uff08\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u306e\u91cd\u307f\uff09\n    pred_ensemble = 0.5 * pred_lgb + 0.2 * pred_xgb + 0.3 * pred_cat\n    pred_class = np.argmax(pred_ensemble, axis=1)\n    accuracy = np.mean(pred_class == y_val)\n    \n    print(f'Fold {fold + 1} Ensemble Accuracy: {accuracy:.4f}')\n    val_scores.append(accuracy)\n\n# \u6700\u7d42\u7d50\u679c\nmean_accuracy = np.mean(val_scores)\nstd_accuracy = np.std(val_scores)\nprint(f'\\n=== Final Results ===')\nprint(f'Mean CV Accuracy: {mean_accuracy:.4f} (+/- {std_accuracy:.4f})')\nprint(f'Target 87%: {\"\u2705 ACHIEVED!!!\" if mean_accuracy >= 0.87 else \"\u274c NOT YET\"}')"}, {"cell_type": "code", "execution_count": null, "id": "cell-6", "metadata": {}, "outputs": [], "source": "# \u30e2\u30c7\u30eb\u4fdd\u5b58\nprint('\\nSaving models...')\n\n# \u5404fold\u306e\u30e2\u30c7\u30eb\u3092\u4fdd\u5b58\nfor i, model in enumerate(models_lgb):\n    with open(f'lgb_model_v9_fold{i}.pkl', 'wb') as f:\n        pickle.dump(model, f)\n\nfor i, model in enumerate(models_xgb):\n    with open(f'xgb_model_v9_fold{i}.pkl', 'wb') as f:\n        pickle.dump(model, f)\n\nfor i, model in enumerate(models_cat):\n    model.save_model(f'cat_model_v9_fold{i}.cbm')\n\n# Label encoder\nwith open('label_encoder_v9.pkl', 'wb') as f:\n    pickle.dump(le, f)\n\nprint('Models saved successfully')\nprint(f'\\n\ud83c\udfaf Final Performance: {mean_accuracy:.4f}')"}, {"cell_type": "code", "execution_count": null, "id": "cell-7", "metadata": {}, "outputs": [], "source": "# CMI\u63a8\u8ad6\u30b5\u30fc\u30d0\u30fc\nimport sys\nsys.path.append('/kaggle/input/cmi-detect-behavior-with-sensor-data')\nfrom kaggle_evaluation.cmi_inference_server import CMIInferenceServer\n\ndef predict(sequence, demographics):\n    \"\"\"CMI\u63a8\u8ad6\u30b5\u30fc\u30d0\u30fc\u7528\u4e88\u6e2c\u95a2\u6570\uff08\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u30d9\u30fc\u30b9\uff09\"\"\"\n    try:\n        # DataFrame\u306b\u5909\u63db\n        df_seq = pd.DataFrame(sequence)\n        \n        # \u9ad8\u5ea6\u306a\u7279\u5fb4\u91cf\u62bd\u51fa\n        features = extract_advanced_features(df_seq)\n        X_feat = pd.DataFrame([features])\n        X_feat = X_feat.fillna(0)\n        X_feat = X_feat.replace([np.inf, -np.inf], 0)\n        \n        # \u5168fold\u306e\u4e88\u6e2c\u3092\u5e73\u5747\n        all_preds = []\n        \n        # LightGBM\u4e88\u6e2c\n        for model in models_lgb:\n            pred = model.predict(X_feat, num_iteration=model.best_iteration)\n            all_preds.append(pred[0])\n        \n        # XGBoost\u4e88\u6e2c\n        dtest = xgb.DMatrix(X_feat)\n        for model in models_xgb:\n            pred = model.predict(dtest)\n            all_preds.append(pred[0])\n        \n        # CatBoost\u4e88\u6e2c\n        for model in models_cat:\n            pred = model.predict_proba(X_feat)\n            all_preds.append(pred[0])\n        \n        # \u91cd\u307f\u4ed8\u304d\u30a2\u30f3\u30b5\u30f3\u30d6\u30eb\uff08\u30c8\u30c3\u30d7\u30bd\u30ea\u30e5\u30fc\u30b7\u30e7\u30f3\u306e\u91cd\u307f\uff09\n        # LightGBM: 50%, XGBoost: 20%, CatBoost: 30%\n        n_lgb = len(models_lgb)\n        n_xgb = len(models_xgb)\n        n_cat = len(models_cat)\n        \n        pred_lgb = np.mean(all_preds[:n_lgb], axis=0)\n        pred_xgb = np.mean(all_preds[n_lgb:n_lgb+n_xgb], axis=0)\n        pred_cat = np.mean(all_preds[n_lgb+n_xgb:], axis=0)\n        \n        pred_final = 0.5 * pred_lgb + 0.2 * pred_xgb + 0.3 * pred_cat\n        \n        # \u4e88\u6e2c\u30af\u30e9\u30b9\n        pred_class = np.argmax(pred_final)\n        pred_gesture = le.inverse_transform([pred_class])[0]\n        \n        return pred_gesture\n        \n    except Exception as e:\n        print(f'Prediction error: {e}')\n        return 'Text on phone'  # Most common class\n\nprint('Starting CMI Inference Server...')\nserver = CMIInferenceServer(predict)\nserver.serve()\nprint('Inference complete')"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 5}