{"cells":[{"cell_type":"markdown","metadata":{},"source":"# CMI Competition - Stable GPU Models Submission\n\n**Using newly trained Stable LSTM + Attention models**\n- CV Accuracy: 19.68% ± 3.41%\n- GPU Training: NVIDIA GeForce RTX 3080 Ti\n- NaN Problem: Resolved\n- Enhanced Features: 332→47 dimensions"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom sklearn.preprocessing import StandardScaler\nimport json\nimport warnings\nwarnings.filterwarnings('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")"},{"cell_type":"markdown","metadata":{},"source":"## Model Architecture - StableAttentionModel"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class StableAttentionModel(nn.Module):\n    \"\"\"\n    Stable LSTM + Attention Model\n    Trained with NaN-safe implementation\n    \"\"\"\n    \n    def __init__(self, input_dim=47, hidden_dim=64, num_classes=18, dropout=0.2):\n        super().__init__()\n        \n        self.input_dim = input_dim\n        self.hidden_dim = hidden_dim\n        self.num_classes = num_classes\n        \n        # Simple LSTM\n        self.lstm = nn.LSTM(\n            input_dim, hidden_dim, \n            batch_first=True, \n            bidirectional=True,\n            dropout=dropout\n        )\n        \n        # Attention mechanism\n        self.attention = nn.Linear(hidden_dim * 2, 1)\n        \n        # Classifier\n        self.classifier = nn.Sequential(\n            nn.Linear(hidden_dim * 2, hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, num_classes)\n        )\n        \n        print(f\"🏗️ Stable Attention Model loaded: {input_dim}→{num_classes} classes\")\n    \n    def forward(self, x):\n        # Input validation and clipping\n        x = torch.clamp(x, -5, 5)\n        \n        # LSTM forward pass\n        lstm_out, _ = self.lstm(x)  # (batch, seq_len, hidden*2)\n        \n        # Attention mechanism\n        attn_scores = self.attention(lstm_out).squeeze(-1)  # (batch, seq_len)\n        attn_scores = torch.clamp(attn_scores, -10, 10)  # Prevent overflow\n        attn_weights = F.softmax(attn_scores, dim=1)  # (batch, seq_len)\n        \n        # Weighted sum\n        context = torch.sum(\n            lstm_out * attn_weights.unsqueeze(-1), dim=1\n        )  # (batch, hidden*2)\n        \n        # Classification\n        output = self.classifier(context)\n        \n        return output, attn_weights"},{"cell_type":"markdown","metadata":{},"source":"## Enhanced Feature Engineering"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def enhanced_feature_engineering(seq_data):\n    \"\"\"\n    Enhanced feature engineering: 332→47 dimensions\n    Same as training pipeline\n    \"\"\"\n    try:\n        # Convert polars to pandas if needed\n        if hasattr(seq_data, 'to_pandas'):\n            seq_data = seq_data.to_pandas()\n        \n        # IMU data (first 7 columns)\n        imu_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_w', 'rot_x', 'rot_y', 'rot_z']\n        imu_data = seq_data[imu_cols].values\n        \n        # Enhanced IMU features\n        acc = imu_data[:, :3]\n        rot = imu_data[:, 3:]\n        \n        acc_mag = np.linalg.norm(acc, axis=1, keepdims=True)\n        \n        # Simple gravity removal\n        gravity_est = np.array([0, 0, 9.81])\n        linear_acc = acc - gravity_est\n        linear_acc_mag = np.linalg.norm(linear_acc, axis=1, keepdims=True)\n        \n        # Angular velocity\n        angular_vel = np.diff(rot, axis=0, prepend=rot[:1])\n        angular_vel_mag = np.linalg.norm(angular_vel, axis=1, keepdims=True)\n        \n        # IMU features: 16 dims\n        imu_enhanced = np.hstack([\n            imu_data,           # 7\n            acc_mag,            # 1\n            linear_acc,         # 3\n            linear_acc_mag,     # 1\n            angular_vel,        # 3\n            angular_vel_mag     # 1\n        ])\n        \n        # TOF aggregation (320→20)\n        tof_cols = [col for col in seq_data.columns if col.startswith('tof_')][:320]\n        if len(tof_cols) >= 320:\n            tof_data = seq_data[tof_cols].values\n            \n            tof_aggregated = []\n            for sensor in range(5):\n                start_idx = sensor * 64\n                end_idx = (sensor + 1) * 64\n                if end_idx <= tof_data.shape[1]:\n                    sensor_data = tof_data[:, start_idx:end_idx]\n                    \n                    sensor_mean = np.mean(sensor_data, axis=1, keepdims=True)\n                    sensor_std = np.std(sensor_data, axis=1, keepdims=True)\n                    sensor_min = np.min(sensor_data, axis=1, keepdims=True)\n                    sensor_max = np.max(sensor_data, axis=1, keepdims=True)\n                    \n                    tof_aggregated.extend([sensor_mean, sensor_std, sensor_min, sensor_max])\n            \n            if len(tof_aggregated) == 20:\n                tof_enhanced = np.hstack(tof_aggregated)  # 20 dims\n            else:\n                tof_enhanced = np.zeros((len(seq_data), 20))\n        else:\n            tof_enhanced = np.zeros((len(seq_data), 20))\n        \n        # Thermal features (5)\n        thm_cols = ['thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5']\n        thermal_data = seq_data[thm_cols].values\n        \n        # Combine: 16 + 20 + 5 = 41 dims\n        enhanced_features = np.hstack([\n            imu_enhanced,    # 16\n            tof_enhanced,    # 20\n            thermal_data     # 5\n        ])\n        \n        # Pad to 47 dims\n        if enhanced_features.shape[1] < 47:\n            padding = np.zeros((enhanced_features.shape[0], 47 - enhanced_features.shape[1]))\n            enhanced_features = np.hstack([enhanced_features, padding])\n        \n        return enhanced_features[:, :47]\n        \n    except Exception as e:\n        print(f\"Feature engineering failed: {e}\")\n        # Fallback: use first 47 columns\n        return seq_data.iloc[:, :47].values if hasattr(seq_data, 'iloc') else seq_data[:, :47]"},{"cell_type":"markdown","metadata":{},"source":"## Load Stable GPU-Trained Models"},{"cell_type":"code","metadata":{},"outputs":[],"source":"def load_stable_gpu_models():\n    \"\"\"Load high-accuracy reference models (57% CV)\"\"\"\n    global models, gesture_names, scaler\n    \n    # Model directory - try new high-accuracy models first\n    model_dir = '/kaggle/input/cmi-reference-models-57cv/'\n    \n    if not os.path.exists(model_dir):\n        model_dir = '/kaggle/input/cmi-enhanced-models-v1/'  # Fallback\n    \n    if not os.path.exists(model_dir):\n        raise FileNotFoundError(f\"Model directory not found\")\n    \n    # Load training summary\n    summary_files = ['reference_training_summary.json', 'stable_training_summary.json']\n    summary = None\n    \n    for summary_file in summary_files:\n        summary_path = os.path.join(model_dir, summary_file)\n        if os.path.exists(summary_path):\n            with open(summary_path, 'r') as f:\n                summary = json.load(f)\n            break\n    \n    if summary is None:\n        raise FileNotFoundError(\"No training summary found\")\n    \n    print(f\"📊 Model Summary:\")\n    print(f\"   Model Type: {summary.get('model_type', 'Unknown')}\")\n    if 'cv_results' in summary:\n        print(f\"   CV Accuracy: {summary['cv_results']['mean_accuracy']:.4f} ± {summary['cv_results']['std_accuracy']:.4f}\")\n    \n    # Load gesture names\n    gesture_names = summary['label_encoder_classes']\n    print(f\"   Gesture Classes: {len(gesture_names)}\")\n    \n    # Load models (try different naming patterns)\n    models = []\n    model_patterns = ['reference_model_fold{}.pth', 'stable_model_fold{}.pth', 'enhanced_fold{}.pth']\n    \n    for pattern in model_patterns:\n        models = []\n        for fold in range(5):\n            model_path = os.path.join(model_dir, pattern.format(fold))\n            if os.path.exists(model_path):\n                print(f\"   Loading {pattern.format(fold)}...\")\n                checkpoint = torch.load(model_path, map_location=device)\n                \n                # Handle different model types\n                if 'ReferenceModel' in checkpoint.get('model_type', ''):\n                    # High-accuracy reference model\n                    config = checkpoint['config']['model']\n                    model = ReferenceModel(**config)\n                else:\n                    # Fallback to stable model\n                    config = checkpoint['config']['model']\n                    model = StableAttentionModel(**config)\n                \n                model.load_state_dict(checkpoint['model_state_dict'])\n                model.to(device)\n                model.eval()\n                models.append(model)\n        \n        if models:\n            print(f\"   Loaded {len(models)} models using pattern: {pattern}\")\n            break\n    \n    if not models:\n        raise RuntimeError(\"No models could be loaded\")\n    \n    # Initialize scaler\n    scaler = StandardScaler()\n    \n    return len(models)\n\n# Define ReferenceModel class for high-accuracy models\nclass ReferenceModel(nn.Module):\n    def __init__(self, input_dim=332, hidden_dim=128, num_classes=18, dropout=0.2):\n        super().__init__()\n        self.input_proj = nn.Sequential(\n            nn.Linear(input_dim, hidden_dim * 2),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim * 2, hidden_dim)\n        )\n        self.lstm1 = nn.LSTM(hidden_dim, hidden_dim, batch_first=True, bidirectional=True, num_layers=2, dropout=dropout)\n        self.lstm2 = nn.LSTM(hidden_dim * 2, hidden_dim, batch_first=True, bidirectional=True, dropout=dropout)\n        self.global_pool = nn.AdaptiveAvgPool1d(1)\n        self.global_max_pool = nn.AdaptiveMaxPool1d(1)\n        self.classifier = nn.Sequential(\n            nn.Linear(hidden_dim * 4, hidden_dim * 2),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim * 2, hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(hidden_dim, num_classes)\n        )\n        print(f\"🚀 Reference Model loaded: {input_dim}→{num_classes} classes\")\n    \n    def forward(self, x):\n        x = self.input_proj(x)\n        x, _ = self.lstm1(x)\n        x, _ = self.lstm2(x)\n        x_t = x.transpose(1, 2)\n        avg_pool = self.global_pool(x_t).squeeze(-1)\n        max_pool = self.global_max_pool(x_t).squeeze(-1)\n        features = torch.cat([avg_pool, max_pool], dim=1)\n        return self.classifier(features)\n\n# Load models\nload_stable_gpu_models()"},{"cell_type":"markdown","metadata":{},"source":"## Prediction Function"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    \"\"\"\n    Predict gesture using stable GPU-trained models\n    3-fold ensemble with enhanced features\n    \"\"\"\n    global scaler_fitted\n    \n    try:\n        # Enhanced feature engineering\n        enhanced_features = enhanced_feature_engineering(sequence)\n        \n        # Data normalization\n        if 'scaler_fitted' not in globals():\n            # Fit scaler on first sequence (approximate)\n            scaler.fit(enhanced_features)\n            scaler_fitted = True\n        \n        normalized_features = scaler.transform(enhanced_features)\n        \n        # Sequence length normalization\n        target_seq_len = 100\n        seq_len = len(normalized_features)\n        \n        if seq_len > target_seq_len:\n            # Downsample\n            indices = np.linspace(0, seq_len-1, target_seq_len, dtype=int)\n            normalized_features = normalized_features[indices]\n        elif seq_len < target_seq_len:\n            # Pad with zeros\n            pad_size = target_seq_len - seq_len\n            padding = np.zeros((pad_size, normalized_features.shape[1]))\n            normalized_features = np.vstack([normalized_features, padding])\n        \n        # Convert to tensor\n        input_tensor = torch.FloatTensor(normalized_features).unsqueeze(0).to(device)  # (1, seq_len, 47)\n        \n        # Ensemble prediction\n        ensemble_probs = []\n        \n        with torch.no_grad():\n            for model in models:\n                outputs, _ = model(input_tensor)\n                probs = F.softmax(outputs, dim=-1)\n                ensemble_probs.append(probs.cpu().numpy())\n        \n        # Average probabilities\n        avg_probs = np.mean(ensemble_probs, axis=0)\n        predicted_class = np.argmax(avg_probs)\n        \n        # Map to gesture name\n        if predicted_class < len(gesture_names):\n            predicted_gesture = gesture_names[predicted_class]\n        else:\n            # Fallback to most common gesture\n            predicted_gesture = \"Neck - scratch\"\n        \n        return predicted_gesture\n        \n    except Exception as e:\n        print(f\"Prediction error: {e}\")\n        # Return most common gesture as fallback\n        return \"Neck - scratch\""},{"cell_type":"markdown","metadata":{},"source":"## Test Prediction Function"},{"cell_type":"code","metadata":{},"outputs":[],"source":"# Import evaluation server\nimport kaggle_evaluation.cmi_inference_server\n\n# Test the prediction function if local test data exists  \nif os.path.exists('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv'):\n    test_df = pl.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\n    test_demographics = pl.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv')\n    \n    # Test on first sequence\n    first_seq_id = test_df['sequence_id'].unique()[0]\n    first_sequence = test_df.filter(pl.col('sequence_id') == first_seq_id)\n    first_demo = test_demographics.filter(pl.col('sequence_id') == first_seq_id)\n    \n    print(f\"Testing on sequence {first_seq_id}...\")\n    result = predict(first_sequence, first_demo)\n    print(f\"Predicted gesture: {result}\")\n    print(f\"\\n✅ Stable GPU models working correctly!\")\nelse:\n    print(\"Competition test data not found - models loaded successfully\")"},{"cell_type":"markdown","metadata":{},"source":"# Competition Evaluation Server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    # Competition evaluation mode\n    inference_server.serve()\nelse:\n    # Local testing mode\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}