{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-22T15:11:42.528324Z","iopub.execute_input":"2025-06-22T15:11:42.528544Z","iopub.status.idle":"2025-06-22T15:11:44.680803Z","shell.execute_reply.started":"2025-06-22T15:11:42.528524Z","shell.execute_reply":"2025-06-22T15:11:44.679957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder, RobustScaler\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.layers import (\n    Conv1D, MaxPooling1D, Dense, Dropout, BatchNormalization,\n    LSTM, Bidirectional, GlobalAveragePooling1D, Input, Concatenate,\n    SeparableConv1D, GlobalMaxPooling1D, MultiHeadAttention, LayerNormalization\n)\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.regularizers import l2\nimport tensorflow as tf\nimport polars as pl\nfrom scipy import signal\nfrom scipy.stats import skew, kurtosis\nimport json\n\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T15:13:23.224200Z","iopub.execute_input":"2025-06-22T15:13:23.224606Z","iopub.status.idle":"2025-06-22T15:13:40.675603Z","shell.execute_reply.started":"2025-06-22T15:13:23.224579Z","shell.execute_reply":"2025-06-22T15:13:40.674751Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. CONFIGURATION & GPU SETUP","metadata":{}},{"cell_type":"code","source":"class Config:\n    RANDOM_STATE = 42\n    N_FOLDS = 5\n    SEQUENCE_PERCENTILE = 95\n    EPOCHS = 100\n    DROP_THERMAL_TOF = True\n    USE_FEATURE_ENGINEERING = True\n    USE_ATTENTION = True\n    USE_SUBJECT_CV = True\n    \n    # OPTIMAL PARAMETERS (from hyperparameter search)\n    OPTIMAL_PARAMS = {\n        'conv1_filters': 64,\n        'conv2_filters': 128,\n        'conv3_filters': 256,\n        'lstm1_units': 128,\n        'lstm2_units': 64,\n        'conv_dropout': 0.35,\n        'lstm_dropout': 0.3,\n        'dense_dropout': 0.4,\n        'dense_size': 512,\n        'learning_rate': 0.0005,\n        'batch_size': 64,\n        'l2_reg': 0.005,\n        'attention_heads': 8\n    }\n\n# GPU Configuration\nprint(\"Setting up GPU configuration...\")\ngpus = tf.config.experimental.list_physical_devices('GPU')\nif gpus:\n    try:\n        for gpu in gpus:\n            tf.config.experimental.set_memory_growth(gpu, True)\n        \n        policy = tf.keras.mixed_precision.Policy('mixed_float16')\n        tf.keras.mixed_precision.set_global_policy(policy)\n        \n        print(f\"✅ GPU acceleration enabled! Found {len(gpus)} GPU(s)\")\n        print(f\"✅ Mixed precision enabled for faster training\")\n        print(f\"GPU devices: {[gpu.name for gpu in gpus]}\")\n        \n    except RuntimeError as e:\n        print(f\"⚠️  GPU setup error: {e}\")\nelse:\n    print(\"⚠️  No GPU found, using CPU\")\n\ntf.random.set_seed(Config.RANDOM_STATE)\nnp.random.seed(Config.RANDOM_STATE)\n\nprint(\"Loading sensor dataset...\")\nroot = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\n\ndf = pd.read_csv(f\"{root}/train.csv\")\nprint(f\"Loaded {len(df):,} rows of sensor frames\")\n\nprint(\"Merging demographic attributes...\")\ndemographics = pd.read_csv(f\"{root}/train_demographics.csv\")\ndf = df.merge(demographics, on=\"subject\", how=\"left\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-22T15:14:14.303171Z","iopub.execute_input":"2025-06-22T15:14:14.303794Z","execution_failed":"2025-06-22T15:14:38.563Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"2. FEATURE ENGINEERING","metadata":{}},{"cell_type":"code","source":"def add_engineered_features(df):\n    \"\"\"Add engineered features to the dataframe\"\"\"\n    if not Config.USE_FEATURE_ENGINEERING:\n        return df\n    \n    print(\"Engineering additional features...\")\n    \n    # Acceleration magnitude\n    if all(col in df.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n        df['acc_magnitude'] = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n    \n    # Gyroscope magnitude\n    if all(col in df.columns for col in ['gyr_x', 'gyr_y', 'gyr_z']):\n        df['gyr_magnitude'] = np.sqrt(df['gyr_x']**2 + df['gyr_y']**2 + df['gyr_z']**2)\n    \n    # Jerk (acceleration derivative)\n    for axis in ['x', 'y', 'z']:\n        if f'acc_{axis}' in df.columns:\n            df[f'jerk_{axis}'] = df.groupby('sequence_id')[f'acc_{axis}'].diff()\n    \n    # Rolling statistics (window of 5)\n    sensor_cols = [c for c in df.columns if c.startswith(('acc_', 'gyr_', 'mag_'))]\n    for col in sensor_cols:\n        if col in df.columns:\n            df[f'{col}_roll_mean'] = df.groupby('sequence_id')[col].rolling(5, center=True).mean().reset_index(0, drop=True)\n            df[f'{col}_roll_std'] = df.groupby('sequence_id')[col].rolling(5, center=True).std().reset_index(0, drop=True)\n    \n    return df\n\n# Apply feature engineering\ndf = add_engineered_features(df)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"3. LABEL ENCODING & FEATURE SELECTION","metadata":{}},{"cell_type":"code","source":"label_encoder = LabelEncoder()\ndf[\"gesture\"] = label_encoder.fit_transform(df[\"gesture\"].astype(str))\nnp.save(\"gesture_classes.npy\", label_encoder.classes_)\n\nprint(\"Gesture label mapping:\")\nfor idx, lab in enumerate(label_encoder.classes_):\n    print(f\"  {idx}: {lab}\")\n\n# Feature selection\nexcluded_cols = {\n    \"gesture\", \"sequence_type\", \"behavior\", \"orientation\",\n    \"row_id\", \"subject\", \"phase\",\n    \"sequence_id\", \"sequence_counter\"\n}\n\nthermal_tof_cols = [c for c in df.columns if c.startswith((\"thm_\", \"tof_\"))]\nif Config.DROP_THERMAL_TOF:\n    excluded_cols.update(thermal_tof_cols)\n    print(f\"Excluding {len(thermal_tof_cols)} thermal/TOF channels\")\n\nfeature_cols = [c for c in df.columns if c not in excluded_cols]\nprint(f\"Using {len(feature_cols)} feature columns\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"4. PREPROCESSING","metadata":{}},{"cell_type":"code","source":"def preprocess_sequence_enhanced(df_seq: pd.DataFrame, feature_columns: list[str], \n                               scaler=None, fit_scaler=True) -> tuple:\n    \"\"\"Enhanced preprocessing with better scaling strategy\"\"\"\n    data = df_seq[feature_columns].copy()\n    \n    # Handle missing values\n    data = data.ffill().bfill().fillna(0.0)\n    \n    # Apply filtering to reduce noise\n    for col in data.select_dtypes(include=[np.number]).columns:\n        data[col] = data[col].rolling(window=3, center=True, min_periods=1).mean()\n    \n    # Robust scaling\n    if scaler is None:\n        scaler = RobustScaler()\n    \n    if fit_scaler:\n        scaled = scaler.fit_transform(data)\n    else:\n        scaled = scaler.transform(data)\n    \n    return scaled.astype(\"float32\"), scaler","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"5. MODEL ARCHITECTURE","metadata":{}},{"cell_type":"code","source":"def create_production_model(input_shape, num_classes):\n    \"\"\"Create production model with optimal parameters\"\"\"\n    \n    params = Config.OPTIMAL_PARAMS\n    \n    with tf.device('/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'):\n        if Config.USE_ATTENTION:\n            # Model with attention mechanism\n            inputs = Input(shape=input_shape, dtype=tf.float16 if tf.config.list_physical_devices('GPU') else tf.float32)\n            \n            # CNN feature extraction\n            x = Conv1D(params['conv1_filters'], 3, activation=\"relu\", padding=\"same\")(inputs)\n            x = BatchNormalization()(x)\n            x = SeparableConv1D(params['conv1_filters'], 3, activation=\"relu\", padding=\"same\")(x)\n            x = MaxPooling1D(2)(x)\n            x = Dropout(params['conv_dropout'])(x)\n            \n            x = Conv1D(params['conv2_filters'], 5, activation=\"relu\", padding=\"same\")(x)\n            x = BatchNormalization()(x)\n            x = SeparableConv1D(params['conv2_filters'], 5, activation=\"relu\", padding=\"same\")(x)\n            x = MaxPooling1D(2)(x)\n            x = Dropout(params['conv_dropout'])(x)\n            \n            x = Conv1D(params['conv3_filters'], 7, activation=\"relu\", padding=\"same\")(x)\n            x = BatchNormalization()(x)\n            x = SeparableConv1D(params['conv3_filters'], 7, activation=\"relu\", padding=\"same\")(x)\n            x = MaxPooling1D(2)(x)\n            x = Dropout(params['conv_dropout'])(x)\n            \n            # BiLSTM layers\n            x = Bidirectional(LSTM(params['lstm1_units'], return_sequences=True, dropout=params['lstm_dropout']))(x)\n            x = Bidirectional(LSTM(params['lstm2_units'], return_sequences=True, dropout=params['lstm_dropout']))(x)\n            \n            # Multi-head attention\n            attention = MultiHeadAttention(num_heads=params['attention_heads'], key_dim=params['lstm2_units'])(x, x)\n            attention = LayerNormalization()(attention + x)\n            \n            # Global pooling\n            global_avg = GlobalAveragePooling1D()(attention)\n            global_max = GlobalMaxPooling1D()(attention)\n            concat = Concatenate()([global_avg, global_max])\n            \n            # Dense layers\n            x = Dense(params['dense_size'], activation=\"relu\", kernel_regularizer=l2(params['l2_reg']))(concat)\n            x = BatchNormalization()(x)\n            x = Dropout(params['dense_dropout'])(x)\n            x = Dense(params['dense_size'] // 2, activation=\"relu\", kernel_regularizer=l2(params['l2_reg']))(x)\n            x = Dropout(params['dense_dropout'] * 0.6)(x)\n            \n            outputs = Dense(num_classes, activation=\"softmax\", dtype='float32')(x)\n            model = Model(inputs, outputs)\n            \n        else:\n            # Sequential model\n            model = Sequential([\n                Conv1D(params['conv1_filters'], 3, activation=\"relu\", input_shape=input_shape, padding=\"same\"),\n                BatchNormalization(),\n                SeparableConv1D(params['conv1_filters'], 3, activation=\"relu\", padding=\"same\"),\n                MaxPooling1D(2),\n                Dropout(params['conv_dropout']),\n\n                Conv1D(params['conv2_filters'], 5, activation=\"relu\", padding=\"same\"),\n                BatchNormalization(),\n                SeparableConv1D(params['conv2_filters'], 5, activation=\"relu\", padding=\"same\"),\n                MaxPooling1D(2),\n                Dropout(params['conv_dropout']),\n\n                Conv1D(params['conv3_filters'], 7, activation=\"relu\", padding=\"same\"),\n                BatchNormalization(),\n                SeparableConv1D(params['conv3_filters'], 7, activation=\"relu\", padding=\"same\"),\n                MaxPooling1D(2),\n                Dropout(params['conv_dropout']),\n\n                Bidirectional(LSTM(params['lstm1_units'], return_sequences=True, dropout=params['lstm_dropout'])),\n                Bidirectional(LSTM(params['lstm2_units'], return_sequences=False, dropout=params['lstm_dropout'])),\n\n                Dense(params['dense_size'], activation=\"relu\", kernel_regularizer=l2(params['l2_reg'])),\n                BatchNormalization(),\n                Dropout(params['dense_dropout']),\n                Dense(params['dense_size'] // 2, activation=\"relu\", kernel_regularizer=l2(params['l2_reg'])),\n                Dropout(params['dense_dropout'] * 0.6),\n\n                Dense(num_classes, activation=\"softmax\", dtype='float32'),\n            ])\n    \n    # Compile with optimal parameters\n    optimizer = Adam(learning_rate=params['learning_rate'])\n    if tf.config.list_physical_devices('GPU') and tf.keras.mixed_precision.global_policy().name == 'mixed_float16':\n        optimizer = tf.keras.mixed_precision.LossScaleOptimizer(optimizer)\n    \n    model.compile(\n        optimizer=optimizer,\n        loss=\"categorical_crossentropy\",\n        metrics=[\"accuracy\"]\n    )\n    \n    return model","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"6. SUBJECT-WISE CROSS-VALIDATION","metadata":{}},{"cell_type":"code","source":"def get_subject_cv_splits(X, y, subjects):\n    \"\"\"Get subject-wise cross-validation splits\"\"\"\n    print(\"Setting up subject-wise cross-validation...\")\n    \n    # Encode subjects to integers\n    unique_subjects = np.unique(subjects)\n    subject_encoder = LabelEncoder()\n    subjects_encoded = subject_encoder.fit_transform(subjects)\n    \n    print(f\"Total subjects: {len(unique_subjects)}\")\n    print(f\"Subject distribution (sequences per subject): {np.bincount(subjects_encoded)}\")\n    \n    # Use GroupKFold\n    gkf = GroupKFold(n_splits=Config.N_FOLDS)\n    splits = list(gkf.split(X, y, groups=subjects_encoded))\n    \n    # Verify no subject leakage\n    for i, (train_idx, val_idx) in enumerate(splits):\n        train_subjects = set(subjects[train_idx])\n        val_subjects = set(subjects[val_idx])\n        overlap = train_subjects & val_subjects\n        if overlap:\n            print(f\"WARNING: Subject overlap in fold {i}: {overlap}\")\n        else:\n            print(f\"Fold {i}: Train subjects: {len(train_subjects)}, Val subjects: {len(val_subjects)}\")\n    \n    return splits","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"7. MAIN TRAINING PIPELINE","metadata":{}},{"cell_type":"code","source":"print(\"Constructing dataset...\")\nseq_groups = df.groupby(\"sequence_id\")\n\n# Prepare sequences, labels, and subject information\nsequences, labels, scalers = [], [], []\nsequence_ids, subjects_list = [], []\n\nfor seq_id, seq in seq_groups:\n    arr, scaler = preprocess_sequence_enhanced(seq, feature_cols, fit_scaler=True)\n    sequences.append(arr)\n    labels.append(seq[\"gesture\"].iloc[0])\n    scalers.append(scaler)\n    sequence_ids.append(seq_id)\n    subjects_list.append(seq[\"subject\"].iloc[0])\n\n# Convert to arrays\nsequence_ids = np.array(sequence_ids)\nsubjects_array = np.array(subjects_list)\n\n# Determine padding length\nseq_lengths = [seq.shape[0] for seq in sequences]\npad_len = int(np.percentile(seq_lengths, Config.SEQUENCE_PERCENTILE))\nprint(f\"{Config.SEQUENCE_PERCENTILE}th-percentile length = {pad_len}\")\nnp.save(\"sequence_maxlen.npy\", pad_len)\n\n# Pad sequences\nX = pad_sequences(sequences, maxlen=pad_len, dtype=\"float32\", padding=\"post\", truncating=\"post\")\ny = np.array(labels)\n\nprint(f\"Dataset shape: {X.shape}, Labels shape: {y.shape}\")\nprint(f\"Number of unique subjects: {len(np.unique(subjects_array))}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"8. PRODUCTION TRAINING WITH OPTIMAL PARAMETERS","metadata":{}},{"cell_type":"code","source":"print(\"\\n\" + \"=\"*60)\nprint(\"TRAINING PRODUCTION MODEL WITH OPTIMAL PARAMETERS\")\nprint(\"=\"*60)\nprint(f\"Optimal parameters: {Config.OPTIMAL_PARAMS}\")\n\nsplits = get_subject_cv_splits(X, y, subjects_array)\ncv_scores = []\nmodels = []\n\nfor fold, (train_idx, val_idx) in enumerate(splits):\n    print(f\"\\n=== FOLD {fold + 1}/{Config.N_FOLDS} ===\")\n    \n    X_train, X_val = X[train_idx], X[val_idx]\n    y_train, y_val = y[train_idx], y[val_idx]\n    \n    # Print subject distribution\n    train_subjects = set(subjects_array[train_idx])\n    val_subjects = set(subjects_array[val_idx])\n    print(f\"Train subjects: {len(train_subjects)}, Val subjects: {len(val_subjects)}\")\n    \n    # Convert to categorical\n    num_classes = len(np.unique(y))\n    y_train_cat = to_categorical(y_train, num_classes=num_classes)\n    y_val_cat = to_categorical(y_val, num_classes=num_classes)\n    \n    # Create model\n    model = create_production_model((X_train.shape[1], X_train.shape[2]), num_classes)\n    print(f\"Model created with {model.count_params():,} parameters\")\n    \n    # Callbacks\n    callbacks = [\n        ReduceLROnPlateau(monitor='val_loss', patience=5, factor=0.5, verbose=1, min_lr=1e-7),\n        EarlyStopping(monitor='val_loss', patience=12, restore_best_weights=True, verbose=1),\n        ModelCheckpoint(f'best_model_fold_{fold}.h5', save_best_only=True, monitor='val_loss')\n    ]\n    \n    # Train model\n    with tf.device('/GPU:0' if tf.config.list_physical_devices('GPU') else '/CPU:0'):\n        print(f\"Training fold {fold+1} on: {'GPU' if tf.config.list_physical_devices('GPU') else 'CPU'}\")\n        history = model.fit(\n            X_train, y_train_cat,\n            epochs=Config.EPOCHS,\n            batch_size=Config.OPTIMAL_PARAMS['batch_size'],\n            validation_data=(X_val, y_val_cat),\n            callbacks=callbacks,\n            verbose=1\n        )\n    \n    # Evaluate fold\n    val_pred = model.predict(X_val, verbose=0)\n    val_pred_labels = np.argmax(val_pred, axis=1)\n    \n    # Calculate metric\n    try:\n        from cmi_2025_metric_copy_for_import import CompetitionMetric\n        cls = label_encoder.classes_\n        val_pred_df = pd.DataFrame({\"gesture\": [cls[i] for i in val_pred_labels]})\n        val_true_df = pd.DataFrame({\"gesture\": [cls[i] for i in y_val]})\n        \n        metric = CompetitionMetric()\n        score = metric.calculate_hierarchical_f1(val_true_df, val_pred_df)\n        cv_scores.append(score)\n        print(f\"Fold {fold + 1} Hierarchical F1: {score:.4f}\")\n    except:\n        accuracy = np.mean(val_pred_labels == y_val)\n        cv_scores.append(accuracy)\n        print(f\"Fold {fold + 1} Accuracy: {accuracy:.4f}\")\n    \n    models.append(model)\n\nprint(f\"\\nFinal Cross-validation scores: {cv_scores}\")\nprint(f\"Mean CV score: {np.mean(cv_scores):.4f} ± {np.std(cv_scores):.4f}\")\n\n# Save best model\nbest_fold = np.argmax(cv_scores)\nbest_model = models[best_fold]\nbest_model.save(\"gesture_cnn_model_enhanced.h5\")\nprint(f\"Best model from fold {best_fold + 1} saved\")\n\n# Save configuration\nwith open('production_config.json', 'w') as f:\n    json.dump({\n        'optimal_parameters': Config.OPTIMAL_PARAMS,\n        'cv_scores': [float(score) for score in cv_scores],\n        'mean_cv_score': float(np.mean(cv_scores)),\n        'std_cv_score': float(np.std(cv_scores)),\n        'best_fold': int(best_fold),\n        'model_parameters': int(best_model.count_params())\n    }, f, indent=2)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}