{"cells":[{"cell_type":"markdown","metadata":{},"source":"# Ultra Advanced Model Submission - 71.86% Validation Accuracy\n\nThis notebook implements the Ultra Advanced model with enhanced features:\n- Enhanced SE blocks with avg+max pooling\n- Multi-scale CNN (kernel sizes 3,5,7)\n- BiLSTM + BiGRU ensemble\n- Attention mechanism\n- Gravity removal using quaternions\n- TOF aggregation features\n- Meta feature extraction","outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.spatial.transform import Rotation as R\nimport joblib\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# For Kaggle environment\nimport polars as pl"},{"cell_type":"code","source":"# Check available files\n!ls -la /kaggle/input/cmi-ultra-advanced-model-7186/","metadata":{},"outputs":[]},{"cell_type":"markdown","metadata":{},"source":"## Model Architecture"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class EnhancedSEBlock(nn.Module):\n    \"\"\"Enhanced Squeeze-and-Excitation block with both avg and max pooling\"\"\"\n    def __init__(self, channels, reduction=8):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\n        self.max_pool = nn.AdaptiveMaxPool1d(1)\n        self.excitation = nn.Sequential(\n            nn.Linear(channels * 2, channels // reduction, bias=False),\n            nn.SiLU(inplace=True),\n            nn.Linear(channels // reduction, channels, bias=False),\n            nn.Sigmoid()\n        )\n    \n    def forward(self, x):\n        b, c, t = x.size()\n        avg_y = self.avg_pool(x).view(b, c)\n        max_y = self.max_pool(x).view(b, c)\n        y = torch.cat([avg_y, max_y], dim=1)\n        y = self.excitation(y).view(b, c, 1)\n        return x * y.expand_as(x)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class ResidualSEBlock(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3):\n        super().__init__()\n        self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n        self.bn1 = nn.BatchNorm1d(out_channels)\n        \n        self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\n        self.bn2 = nn.BatchNorm1d(out_channels)\n        \n        self.se = EnhancedSEBlock(out_channels, reduction=8)\n        \n        self.shortcut = nn.Sequential()\n        if in_channels != out_channels:\n            self.shortcut = nn.Sequential(\n                nn.Conv1d(in_channels, out_channels, 1, bias=False),\n                nn.BatchNorm1d(out_channels)\n            )\n        \n        self.pool = nn.MaxPool1d(pool_size)\n        self.dropout = nn.Dropout(dropout)\n        \n    def forward(self, x):\n        shortcut = self.shortcut(x)\n        \n        out = F.relu(self.bn1(self.conv1(x)))\n        out = self.bn2(self.conv2(out))\n        \n        out = self.se(out)\n        \n        out += shortcut\n        out = F.relu(out)\n        \n        out = self.pool(out)\n        out = self.dropout(out)\n        \n        return out"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class MetaFeatureExtractor(nn.Module):\n    \"\"\"Extract statistical meta-features from input sequence\"\"\"\n    def forward(self, x):\n        # x shape: (B, L, C)\n        mean = torch.mean(x, dim=1)\n        std = torch.std(x, dim=1)\n        max_val, _ = torch.max(x, dim=1)\n        min_val, _ = torch.min(x, dim=1)\n        \n        # Calculate slope\n        seq_len = x.size(1)\n        if seq_len > 1:\n            slope = (x[:, -1, :] - x[:, 0, :]) / (seq_len - 1)\n        else:\n            slope = torch.zeros_like(x[:, 0, :])\n        \n        # Additional features\n        median = torch.median(x, dim=1)[0]\n        \n        return torch.cat([mean, std, max_val, min_val, slope, median], dim=1)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class GaussianNoise(nn.Module):\n    \"\"\"Gaussian noise regularization\"\"\"\n    def __init__(self, stddev):\n        super().__init__()\n        self.stddev = stddev\n        \n    def forward(self, x):\n        if self.training:\n            noise = torch.randn_like(x) * self.stddev\n            return x + noise\n        return x\n\nclass AttentionLayer(nn.Module):\n    \"\"\"Attention pooling layer\"\"\"\n    def __init__(self, hidden_dim):\n        super().__init__()\n        self.attention = nn.Linear(hidden_dim, 1)\n        \n    def forward(self, x):\n        # x shape: (batch, seq_len, hidden_dim)\n        scores = torch.tanh(self.attention(x))\n        weights = F.softmax(scores.squeeze(-1), dim=1)\n        context = torch.sum(x * weights.unsqueeze(-1), dim=1)\n        return context"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class UltraAdvancedModel(nn.Module):\n    \"\"\"Ultra Advanced Model combining best practices from LB 0.77 and current Advanced Model\"\"\"\n    def __init__(self, num_features, num_classes):\n        super().__init__()\n        \n        # Meta feature extractor\n        self.meta_extractor = MetaFeatureExtractor()\n        meta_features = 6  # mean, std, max, min, slope, median\n        self.meta_dense = nn.Sequential(\n            nn.Linear(meta_features * num_features, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Dropout(0.2)\n        )\n        \n        # Two-branch architecture from LB 0.77\n        # IMU branch (7 features)\n        self.imu_branch = nn.Sequential(\n            ResidualSEBlock(7, 64, 3, dropout=0.1),\n            ResidualSEBlock(64, 128, 5, dropout=0.1)\n        )\n        \n        # TOF/THM branch (5 aggregated features)\n        self.tof_branch = nn.Sequential(\n            nn.Conv1d(5, 64, 3, padding=1, bias=False),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.MaxPool1d(2),\n            nn.Dropout(0.2),\n            nn.Conv1d(64, 128, 3, padding=1, bias=False),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.MaxPool1d(2),\n            nn.Dropout(0.2)\n        )\n        \n        # Bidirectional LSTM and GRU ensemble\n        combined_features = 256  # 128 + 128 from two branches\n        self.bilstm = nn.LSTM(\n            combined_features, 128, \n            num_layers=2, batch_first=True, \n            bidirectional=True, dropout=0.2\n        )\n        \n        self.bigru = nn.GRU(\n            combined_features, 128,\n            num_layers=2, batch_first=True,\n            bidirectional=True, dropout=0.2\n        )\n        \n        # Gaussian noise for regularization\n        self.noise = GaussianNoise(0.09)\n        \n        # Dense processing for merged features\n        self.dense_merged = nn.Sequential(\n            nn.Linear(combined_features, 16),\n            nn.ELU()\n        )\n        \n        # Attention pooling\n        rnn_hidden = 256 * 2 + 16  # BiLSTM + BiGRU + dense\n        self.attention_pooling = AttentionLayer(rnn_hidden)\n        \n        # Final classifier with meta features\n        self.classifier = nn.Sequential(\n            nn.Linear(rnn_hidden + 64, 512),  # RNN features + meta features\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, num_classes)\n        )\n        \n    def forward(self, x):\n        # x shape: (batch, seq_len, features)\n        batch_size = x.size(0)\n        \n        # Extract meta features\n        meta = self.meta_extractor(x)\n        meta_proj = self.meta_dense(meta)\n        \n        # Split features\n        imu_features = x[:, :, :7].transpose(1, 2)  # (B, 7, L)\n        tof_features = x[:, :, 7:].transpose(1, 2)  # (B, 5, L)\n        \n        # Process branches\n        imu_out = self.imu_branch(imu_features)  # (B, 128, L/4)\n        tof_out = self.tof_branch(tof_features)  # (B, 128, L/4)\n        \n        # Ensure same sequence length\n        min_len = min(imu_out.size(2), tof_out.size(2))\n        imu_out = imu_out[:, :, :min_len]\n        tof_out = tof_out[:, :, :min_len]\n        \n        # Merge branches\n        merged = torch.cat([imu_out, tof_out], dim=1)  # (B, 256, L/4)\n        merged = merged.transpose(1, 2)  # (B, L/4, 256)\n        \n        # RNN processing\n        lstm_out, _ = self.bilstm(merged)  # (B, L/4, 256)\n        gru_out, _ = self.bigru(merged)   # (B, L/4, 256)\n        \n        # Noise and dense processing\n        noise_out = self.noise(merged)\n        dense_out = self.dense_merged(noise_out)  # (B, L/4, 16)\n        \n        # Concatenate all features\n        combined = torch.cat([lstm_out, gru_out, dense_out], dim=2)  # (B, L/4, 528)\n        \n        # Attention pooling\n        pooled = self.attention_pooling(combined)  # (B, 528)\n        \n        # Combine with meta features\n        final_features = torch.cat([pooled, meta_proj], dim=1)\n        \n        # Classification\n        output = self.classifier(final_features)\n        \n        return output"},{"cell_type":"markdown","metadata":{},"source":"## Utility Functions"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def remove_gravity_from_acc(acc_data, rot_data):\n    \"\"\"Remove gravity component from accelerometer data using quaternions\"\"\"\n    num_samples = acc_data.shape[0]\n    linear_accel = np.zeros_like(acc_data)\n    \n    gravity_world = np.array([0, 0, 9.81])\n    \n    for i in range(num_samples):\n        if np.all(np.isnan(rot_data[i])) or np.all(np.isclose(rot_data[i], 0)):\n            linear_accel[i, :] = acc_data[i, :]\n            continue\n            \n        try:\n            rotation = R.from_quat(rot_data[i])\n            gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\n            linear_accel[i, :] = acc_data[i, :] - gravity_sensor_frame\n        except ValueError:\n            linear_accel[i, :] = acc_data[i, :]\n    \n    return linear_accel\n\ndef preprocess_sequence(df_seq, scaler, maxlen=103):\n    \"\"\"Preprocess a single sequence with all features from LB 0.77\"\"\"\n    \n    # Original acceleration magnitude\n    df_seq['acc_mag'] = np.sqrt(df_seq['acc_x']**2 + df_seq['acc_y']**2 + df_seq['acc_z']**2)\n    df_seq['rot_angle'] = 2 * np.arccos(df_seq['rot_w'].clip(-1, 1))\n    \n    # Derivatives\n    df_seq['acc_mag_jerk'] = df_seq['acc_mag'].diff().fillna(0)\n    df_seq['rot_angle_vel'] = df_seq['rot_angle'].diff().fillna(0)\n    \n    # Remove gravity\n    acc_data = df_seq[['acc_x', 'acc_y', 'acc_z']].values\n    rot_data = df_seq[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\n    linear_accel = remove_gravity_from_acc(acc_data, rot_data)\n    \n    df_seq['linear_acc_x'] = linear_accel[:, 0]\n    df_seq['linear_acc_y'] = linear_accel[:, 1]\n    df_seq['linear_acc_z'] = linear_accel[:, 2]\n    df_seq['linear_acc_mag'] = np.sqrt(np.sum(linear_accel**2, axis=1))\n    df_seq['linear_acc_mag_jerk'] = df_seq['linear_acc_mag'].diff().fillna(0)\n    \n    # TOF aggregation\n    for i in range(1, 6):\n        pixel_cols = [f\"tof_{i}_v{p}\" for p in range(64)]\n        tof_data = df_seq[pixel_cols].replace(-1, np.nan)\n        df_seq[f'tof_{i}_mean'] = tof_data.mean(axis=1)\n        df_seq[f'tof_{i}_std'] = tof_data.std(axis=1)\n        df_seq[f'tof_{i}_min'] = tof_data.min(axis=1)\n        df_seq[f'tof_{i}_max'] = tof_data.max(axis=1)\n        df_seq[f'tof_{i}_range'] = df_seq[f'tof_{i}_max'] - df_seq[f'tof_{i}_min']\n    \n    # Select features\n    imu_features = ['linear_acc_x', 'linear_acc_y', 'linear_acc_z',\n                   'rot_x', 'rot_y', 'rot_z', 'rot_w']\n    \n    tof_features = []\n    for i in range(1, 6):\n        # Use mean as primary TOF feature (as in LB 0.77)\n        tof_features.append(f'tof_{i}_mean')\n    \n    all_features = imu_features + tof_features\n    \n    # Extract sequence\n    seq_data = df_seq[all_features].fillna(0).values\n    \n    # Normalize\n    seq_data = scaler.transform(seq_data)\n    \n    # Pad or truncate\n    if len(seq_data) > maxlen:\n        seq_data = seq_data[:maxlen]\n    elif len(seq_data) < maxlen:\n        pad_width = ((0, maxlen - len(seq_data)), (0, 0))\n        seq_data = np.pad(seq_data, pad_width, mode='constant')\n    \n    return seq_data"},{"cell_type":"markdown","metadata":{},"source":"## Load Model and Artifacts"},{"cell_type":"code","metadata":{},"outputs":[],"source":"# Set device\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Load artifacts\nMODEL_PATH = '/kaggle/input/cmi-ultra-advanced-model-7186/'\n\n# Load saved artifacts\ngesture_classes = np.load(f'{MODEL_PATH}gesture_classes_ultra.npy')\nmaxlen = int(np.load(f'{MODEL_PATH}sequence_maxlen_ultra.npy'))\nscaler = joblib.load(f'{MODEL_PATH}scaler_ultra.pkl')\n\nprint(f\"Number of classes: {len(gesture_classes)}\")\nprint(f\"Max sequence length: {maxlen}\")\nprint(f\"Classes: {gesture_classes}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Initialize model\nnum_features = 12  # 7 IMU + 5 TOF\nnum_classes = len(gesture_classes)\n\nmodel = UltraAdvancedModel(num_features, num_classes)\nmodel.load_state_dict(torch.load(f'{MODEL_PATH}ultra_advanced_model.pth', map_location=device))\nmodel.to(device)\nmodel.eval()\n\nprint(\"Model loaded successfully!\")"},{"cell_type":"markdown","metadata":{},"source":"## Prediction Function"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    \"\"\"\n    Make prediction for a single sequence using Ultra Advanced Model\n    \"\"\"\n    # Convert to pandas\n    df_seq = sequence.to_pandas()\n    \n    # Preprocess sequence\n    seq_data = preprocess_sequence(df_seq, scaler, maxlen)\n    \n    # Convert to tensor\n    seq_tensor = torch.FloatTensor(seq_data).unsqueeze(0).to(device)\n    \n    # Make prediction\n    with torch.no_grad():\n        logits = model(seq_tensor)\n        pred_idx = torch.argmax(logits, dim=1).item()\n    \n    # Return gesture name\n    return str(gesture_classes[pred_idx])"},{"cell_type":"markdown","metadata":{},"source":"## Kaggle Evaluation Server"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Import Kaggle evaluation module\nimport kaggle_evaluation.cmi_inference_server\n\n# Create inference server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\n# Check if running in Kaggle competition environment\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print(\"Running in Kaggle competition environment\")\n    inference_server.serve()\nelse:\n    print(\"Running local gateway for testing\")\n    # Run local test\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.0"}},"nbformat":4,"nbformat_minor":4}