{"cells":[{"cell_type":"markdown","metadata":{},"source":"# Ultra Advanced Model Submission - 71.86% Validation Accuracy\\n\\nThis notebook implements the Ultra Advanced model with enhanced features:\\n- Enhanced SE blocks with avg+max pooling\\n- Multi-scale CNN (kernel sizes 3,5,7)\\n- BiLSTM + BiGRU ensemble\\n- Attention mechanism\\n- Gravity removal using quaternions\\n- TOF aggregation features\\n- Meta feature extraction"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Check available files\\n!ls -la /kaggle/input/cmi-ultra-advanced-model-7186/"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\\nimport numpy as np\\nimport pandas as pd\\nimport torch\\nimport torch.nn as nn\\nimport torch.nn.functional as F\\nfrom sklearn.preprocessing import StandardScaler\\nfrom scipy.spatial.transform import Rotation as R\\nimport joblib\\nimport warnings\\nwarnings.filterwarnings('ignore')\\n\\n# For Kaggle environment\\nimport polars as pl"},{"cell_type":"markdown","metadata":{},"source":"## Model Architecture"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class EnhancedSEBlock(nn.Module):\\n    \\\"\\\"\\\"Enhanced Squeeze-and-Excitation block with both avg and max pooling\\\"\\\"\\\"\\n    def __init__(self, channels, reduction=8):\\n        super().__init__()\\n        self.avg_pool = nn.AdaptiveAvgPool1d(1)\\n        self.max_pool = nn.AdaptiveMaxPool1d(1)\\n        self.excitation = nn.Sequential(\\n            nn.Linear(channels * 2, channels // reduction, bias=False),\\n            nn.SiLU(inplace=True),\\n            nn.Linear(channels // reduction, channels, bias=False),\\n            nn.Sigmoid()\\n        )\\n    \\n    def forward(self, x):\\n        b, c, t = x.size()\\n        avg_y = self.avg_pool(x).view(b, c)\\n        max_y = self.max_pool(x).view(b, c)\\n        y = torch.cat([avg_y, max_y], dim=1)\\n        y = self.excitation(y).view(b, c, 1)\\n        return x * y.expand_as(x)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class ResidualSEBlock(nn.Module):\\n    def __init__(self, in_channels, out_channels, kernel_size, pool_size=2, dropout=0.3):\\n        super().__init__()\\n        self.conv1 = nn.Conv1d(in_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\\n        self.bn1 = nn.BatchNorm1d(out_channels)\\n        \\n        self.conv2 = nn.Conv1d(out_channels, out_channels, kernel_size, padding=kernel_size//2, bias=False)\\n        self.bn2 = nn.BatchNorm1d(out_channels)\\n        \\n        self.se = EnhancedSEBlock(out_channels, reduction=8)\\n        \\n        self.shortcut = nn.Sequential()\\n        if in_channels != out_channels:\\n            self.shortcut = nn.Sequential(\\n                nn.Conv1d(in_channels, out_channels, 1, bias=False),\\n                nn.BatchNorm1d(out_channels)\\n            )\\n        \\n        self.pool = nn.MaxPool1d(pool_size)\\n        self.dropout = nn.Dropout(dropout)\\n        \\n    def forward(self, x):\\n        shortcut = self.shortcut(x)\\n        \\n        out = F.relu(self.bn1(self.conv1(x)))\\n        out = self.bn2(self.conv2(out))\\n        \\n        out = self.se(out)\\n        \\n        out += shortcut\\n        out = F.relu(out)\\n        \\n        out = self.pool(out)\\n        out = self.dropout(out)\\n        \\n        return out"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class MetaFeatureExtractor(nn.Module):\\n    \\\"\\\"\\\"Extract statistical meta-features from input sequence\\\"\\\"\\\"\\n    def forward(self, x):\\n        # x shape: (B, L, C)\\n        mean = torch.mean(x, dim=1)\\n        std = torch.std(x, dim=1)\\n        max_val, _ = torch.max(x, dim=1)\\n        min_val, _ = torch.min(x, dim=1)\\n        \\n        # Calculate slope\\n        seq_len = x.size(1)\\n        if seq_len > 1:\\n            slope = (x[:, -1, :] - x[:, 0, :]) / (seq_len - 1)\\n        else:\\n            slope = torch.zeros_like(x[:, 0, :])\\n        \\n        # Additional features\\n        median = torch.median(x, dim=1)[0]\\n        \\n        return torch.cat([mean, std, max_val, min_val, slope, median], dim=1)"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class GaussianNoise(nn.Module):\\n    \\\"\\\"\\\"Gaussian noise regularization\\\"\\\"\\\"\\n    def __init__(self, stddev):\\n        super().__init__()\\n        self.stddev = stddev\\n        \\n    def forward(self, x):\\n        if self.training:\\n            noise = torch.randn_like(x) * self.stddev\\n            return x + noise\\n        return x\\n\\nclass AttentionLayer(nn.Module):\\n    \\\"\\\"\\\"Attention pooling layer\\\"\\\"\\\"\\n    def __init__(self, hidden_dim):\\n        super().__init__()\\n        self.attention = nn.Linear(hidden_dim, 1)\\n        \\n    def forward(self, x):\\n        # x shape: (batch, seq_len, hidden_dim)\\n        scores = torch.tanh(self.attention(x))\\n        weights = F.softmax(scores.squeeze(-1), dim=1)\\n        context = torch.sum(x * weights.unsqueeze(-1), dim=1)\\n        return context"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"class UltraAdvancedModel(nn.Module):\\n    \\\"\\\"\\\"Ultra Advanced Model combining best practices from LB 0.77 and current Advanced Model\\\"\\\"\\\"\\n    def __init__(self, num_features, num_classes):\\n        super().__init__()\\n        \\n        # Meta feature extractor\\n        self.meta_extractor = MetaFeatureExtractor()\\n        meta_features = 6  # mean, std, max, min, slope, median\\n        self.meta_dense = nn.Sequential(\\n            nn.Linear(meta_features * num_features, 64),\\n            nn.BatchNorm1d(64),\\n            nn.ReLU(),\\n            nn.Dropout(0.2)\\n        )\\n        \\n        # Two-branch architecture from LB 0.77\\n        # IMU branch (7 features)\\n        self.imu_branch = nn.Sequential(\\n            ResidualSEBlock(7, 64, 3, dropout=0.1),\\n            ResidualSEBlock(64, 128, 5, dropout=0.1)\\n        )\\n        \\n        # TOF/THM branch (5 aggregated features)\\n        self.tof_branch = nn.Sequential(\\n            nn.Conv1d(5, 64, 3, padding=1, bias=False),\\n            nn.BatchNorm1d(64),\\n            nn.ReLU(),\\n            nn.MaxPool1d(2),\\n            nn.Dropout(0.2),\\n            nn.Conv1d(64, 128, 3, padding=1, bias=False),\\n            nn.BatchNorm1d(128),\\n            nn.ReLU(),\\n            nn.MaxPool1d(2),\\n            nn.Dropout(0.2)\\n        )\\n        \\n        # Bidirectional LSTM and GRU ensemble\\n        combined_features = 256  # 128 + 128 from two branches\\n        self.bilstm = nn.LSTM(\\n            combined_features, 128, \\n            num_layers=2, batch_first=True, \\n            bidirectional=True, dropout=0.2\\n        )\\n        \\n        self.bigru = nn.GRU(\\n            combined_features, 128,\\n            num_layers=2, batch_first=True,\\n            bidirectional=True, dropout=0.2\\n        )\\n        \\n        # Gaussian noise for regularization\\n        self.noise = GaussianNoise(0.09)\\n        \\n        # Dense processing for merged features\\n        self.dense_merged = nn.Sequential(\\n            nn.Linear(combined_features, 16),\\n            nn.ELU()\\n        )\\n        \\n        # Attention pooling\\n        rnn_hidden = 256 * 2 + 16  # BiLSTM + BiGRU + dense\\n        self.attention_pooling = AttentionLayer(rnn_hidden)\\n        \\n        # Final classifier with meta features\\n        self.classifier = nn.Sequential(\\n            nn.Linear(rnn_hidden + 64, 512),  # RNN features + meta features\\n            nn.BatchNorm1d(512),\\n            nn.ReLU(),\\n            nn.Dropout(0.5),\\n            nn.Linear(512, 256),\\n            nn.BatchNorm1d(256),\\n            nn.ReLU(),\\n            nn.Dropout(0.3),\\n            nn.Linear(256, num_classes)\\n        )\\n        \\n    def forward(self, x):\\n        # x shape: (batch, seq_len, features)\\n        batch_size = x.size(0)\\n        \\n        # Extract meta features\\n        meta = self.meta_extractor(x)\\n        meta_proj = self.meta_dense(meta)\\n        \\n        # Split features\\n        imu_features = x[:, :, :7].transpose(1, 2)  # (B, 7, L)\\n        tof_features = x[:, :, 7:].transpose(1, 2)  # (B, 5, L)\\n        \\n        # Process branches\\n        imu_out = self.imu_branch(imu_features)  # (B, 128, L/4)\\n        tof_out = self.tof_branch(tof_features)  # (B, 128, L/4)\\n        \\n        # Ensure same sequence length\\n        min_len = min(imu_out.size(2), tof_out.size(2))\\n        imu_out = imu_out[:, :, :min_len]\\n        tof_out = tof_out[:, :, :min_len]\\n        \\n        # Merge branches\\n        merged = torch.cat([imu_out, tof_out], dim=1)  # (B, 256, L/4)\\n        merged = merged.transpose(1, 2)  # (B, L/4, 256)\\n        \\n        # RNN processing\\n        lstm_out, _ = self.bilstm(merged)  # (B, L/4, 256)\\n        gru_out, _ = self.bigru(merged)   # (B, L/4, 256)\\n        \\n        # Noise and dense processing\\n        noise_out = self.noise(merged)\\n        dense_out = self.dense_merged(noise_out)  # (B, L/4, 16)\\n        \\n        # Concatenate all features\\n        combined = torch.cat([lstm_out, gru_out, dense_out], dim=2)  # (B, L/4, 528)\\n        \\n        # Attention pooling\\n        pooled = self.attention_pooling(combined)  # (B, 528)\\n        \\n        # Combine with meta features\\n        final_features = torch.cat([pooled, meta_proj], dim=1)\\n        \\n        # Classification\\n        output = self.classifier(final_features)\\n        \\n        return output"},{"cell_type":"markdown","metadata":{},"source":"## Utility Functions"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def remove_gravity_from_acc(acc_data, rot_data):\\n    \\\"\\\"\\\"Remove gravity component from accelerometer data using quaternions\\\"\\\"\\\"\\n    num_samples = acc_data.shape[0]\\n    linear_accel = np.zeros_like(acc_data)\\n    \\n    gravity_world = np.array([0, 0, 9.81])\\n    \\n    for i in range(num_samples):\\n        if np.all(np.isnan(rot_data[i])) or np.all(np.isclose(rot_data[i], 0)):\\n            linear_accel[i, :] = acc_data[i, :]\\n            continue\\n            \\n        try:\\n            rotation = R.from_quat(rot_data[i])\\n            gravity_sensor_frame = rotation.apply(gravity_world, inverse=True)\\n            linear_accel[i, :] = acc_data[i, :] - gravity_sensor_frame\\n        except ValueError:\\n            linear_accel[i, :] = acc_data[i, :]\\n    \\n    return linear_accel\\n\\ndef preprocess_sequence(df_seq, scaler, maxlen=103):\\n    \\\"\\\"\\\"Preprocess a single sequence with all features from LB 0.77\\\"\\\"\\\"\\n    \\n    # Original acceleration magnitude\\n    df_seq['acc_mag'] = np.sqrt(df_seq['acc_x']**2 + df_seq['acc_y']**2 + df_seq['acc_z']**2)\\n    df_seq['rot_angle'] = 2 * np.arccos(df_seq['rot_w'].clip(-1, 1))\\n    \\n    # Derivatives\\n    df_seq['acc_mag_jerk'] = df_seq['acc_mag'].diff().fillna(0)\\n    df_seq['rot_angle_vel'] = df_seq['rot_angle'].diff().fillna(0)\\n    \\n    # Remove gravity\\n    acc_data = df_seq[['acc_x', 'acc_y', 'acc_z']].values\\n    rot_data = df_seq[['rot_x', 'rot_y', 'rot_z', 'rot_w']].values\\n    linear_accel = remove_gravity_from_acc(acc_data, rot_data)\\n    \\n    df_seq['linear_acc_x'] = linear_accel[:, 0]\\n    df_seq['linear_acc_y'] = linear_accel[:, 1]\\n    df_seq['linear_acc_z'] = linear_accel[:, 2]\\n    df_seq['linear_acc_mag'] = np.sqrt(np.sum(linear_accel**2, axis=1))\\n    df_seq['linear_acc_mag_jerk'] = df_seq['linear_acc_mag'].diff().fillna(0)\\n    \\n    # TOF aggregation\\n    for i in range(1, 6):\\n        pixel_cols = [f\\\"tof_{i}_v{p}\\\" for p in range(64)]\\n        tof_data = df_seq[pixel_cols].replace(-1, np.nan)\\n        df_seq[f'tof_{i}_mean'] = tof_data.mean(axis=1)\\n        df_seq[f'tof_{i}_std'] = tof_data.std(axis=1)\\n        df_seq[f'tof_{i}_min'] = tof_data.min(axis=1)\\n        df_seq[f'tof_{i}_max'] = tof_data.max(axis=1)\\n        df_seq[f'tof_{i}_range'] = df_seq[f'tof_{i}_max'] - df_seq[f'tof_{i}_min']\\n    \\n    # Select features\\n    imu_features = ['linear_acc_x', 'linear_acc_y', 'linear_acc_z',\\n                   'rot_x', 'rot_y', 'rot_z', 'rot_w']\\n    \\n    tof_features = []\\n    for i in range(1, 6):\\n        # Use mean as primary TOF feature (as in LB 0.77)\\n        tof_features.append(f'tof_{i}_mean')\\n    \\n    all_features = imu_features + tof_features\\n    \\n    # Extract sequence\\n    seq_data = df_seq[all_features].fillna(0).values\\n    \\n    # Normalize\\n    seq_data = scaler.transform(seq_data)\\n    \\n    # Pad or truncate\\n    if len(seq_data) > maxlen:\\n        seq_data = seq_data[:maxlen]\\n    elif len(seq_data) < maxlen:\\n        pad_width = ((0, maxlen - len(seq_data)), (0, 0))\\n        seq_data = np.pad(seq_data, pad_width, mode='constant')\\n    \\n    return seq_data"},{"cell_type":"markdown","metadata":{},"source":"## Load Model and Artifacts"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Set device\\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\\nprint(f\\\"Using device: {device}\\\")\\n\\n# Load artifacts\\nMODEL_PATH = '/kaggle/input/cmi-ultra-advanced-model-7186/'\\n\\n# Load saved artifacts with allow_pickle=True\\ngesture_classes = np.load(f'{MODEL_PATH}gesture_classes_ultra.npy', allow_pickle=True)\\nmaxlen = int(np.load(f'{MODEL_PATH}sequence_maxlen_ultra.npy', allow_pickle=True))\\nscaler = joblib.load(f'{MODEL_PATH}scaler_ultra.pkl')\\n\\nprint(f\\\"Number of classes: {len(gesture_classes)}\\\")\\nprint(f\\\"Max sequence length: {maxlen}\\\")\\nprint(f\\\"Classes: {gesture_classes}\\\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Initialize model\\nnum_features = 12  # 7 IMU + 5 TOF\\nnum_classes = len(gesture_classes)\\n\\nmodel = UltraAdvancedModel(num_features, num_classes)\\nmodel.load_state_dict(torch.load(f'{MODEL_PATH}ultra_advanced_model.pth', map_location=device))\\nmodel.to(device)\\nmodel.eval()\\n\\nprint(\\\"Model loaded successfully!\\\")"},{"cell_type":"markdown","metadata":{},"source":"## Prediction Function"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\\n    \\\"\\\"\\\"\\n    Make prediction for a single sequence using Ultra Advanced Model\\n    \\\"\\\"\\\"\\n    # Convert to pandas\\n    df_seq = sequence.to_pandas()\\n    \\n    # Preprocess sequence\\n    seq_data = preprocess_sequence(df_seq, scaler, maxlen)\\n    \\n    # Convert to tensor\\n    seq_tensor = torch.FloatTensor(seq_data).unsqueeze(0).to(device)\\n    \\n    # Make prediction\\n    with torch.no_grad():\\n        logits = model(seq_tensor)\\n        pred_idx = torch.argmax(logits, dim=1).item()\\n    \\n    # Return gesture name\\n    return str(gesture_classes[pred_idx])"},{"cell_type":"markdown","metadata":{},"source":"## Kaggle Evaluation Server"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Import Kaggle evaluation module\\nimport kaggle_evaluation.cmi_inference_server\\n\\n# Create inference server\\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\\n\\n# Check if running in Kaggle competition environment\\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\\n    print(\\\"Running in Kaggle competition environment\\\")\\n    inference_server.serve()\\nelse:\\n    print(\\\"Running local gateway for testing\\\")\\n    # Run local test\\n    inference_server.run_local_gateway(\\n        data_paths=(\\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\\n        )\\n    )"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":4}