{"cells": [{"cell_type": "markdown", "metadata": {}, "source": "# CMI V7 - LSTM Deep Learning Model\\n\\n**Target: 87%+ Validation Accuracy**\\n- Simplified LSTM approach for time series data\\n- Ensemble with traditional ML models"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "import numpy as np\nimport pandas as pd\nimport pickle\nimport gc\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Deep learning imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\n# ML imports\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split\n\nprint(f'TensorFlow version: {tf.__version__}')\nprint(f'GPU available: {tf.config.list_physical_devices(\"GPU\")}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Load data\nprint('Loading training data...')\ntrain_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\n# Get gesture labels\ngesture_labels = sorted(train_df['gesture'].unique())\nnum_classes = len(gesture_labels)\n\nprint(f'Number of classes: {num_classes}')\nprint(f'Total samples: {len(train_df)}')\nprint(f'Unique sequences: {train_df[\"sequence_id\"].nunique()}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "def prepare_sequence_data(df_seq):\n    \"\"\"Prepare sequence data for LSTM\"\"\"\n    # Select sensor columns\n    sensor_cols = ['acc_x', 'acc_y', 'acc_z', 'rot_x', 'rot_y', 'rot_z', 'rot_w']\n    \n    # Add temperature sensors if available\n    for i in range(1, 6):\n        col = f'thm_{i}'\n        if col in df_seq.columns:\n            sensor_cols.append(col)\n    \n    # Get sensor data\n    sequence_data = df_seq[sensor_cols].values\n    \n    return sequence_data\n\ndef pad_sequences_custom(sequences, maxlen=None, padding='post'):\n    \"\"\"Pad sequences to same length\"\"\"\n    if maxlen is None:\n        maxlen = max(len(seq) for seq in sequences)\n    \n    n_features = sequences[0].shape[1]\n    padded = np.zeros((len(sequences), maxlen, n_features))\n    \n    for i, seq in enumerate(sequences):\n        seq_len = min(len(seq), maxlen)\n        if padding == 'post':\n            padded[i, :seq_len] = seq[:seq_len]\n        else:\n            padded[i, -seq_len:] = seq[:seq_len]\n    \n    return padded"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Prepare data for LSTM\nprint('Preparing sequences for LSTM...')\n\nsequences = []\nlabels = []\nsequence_ids = train_df['sequence_id'].unique()\n\n# Encode labels\nle = LabelEncoder()\nle.fit(gesture_labels)\n\nfor seq_id in sequence_ids[:1000]:  # Use subset for faster training\n    seq_data = train_df[train_df['sequence_id'] == seq_id]\n    \n    # Get sequence\n    sequence = prepare_sequence_data(seq_data)\n    sequences.append(sequence)\n    \n    # Get label\n    gesture = seq_data['gesture'].iloc[0]\n    label = le.transform([gesture])[0]\n    labels.append(label)\n\n# Pad sequences\nX_sequences = pad_sequences_custom(sequences, maxlen=100)\ny_labels = np.array(labels)\n\nprint(f'Sequence shape: {X_sequences.shape}')\nprint(f'Labels shape: {y_labels.shape}')\n\n# Normalize data\nscaler = StandardScaler()\nn_samples, n_timesteps, n_features = X_sequences.shape\nX_flat = X_sequences.reshape(-1, n_features)\nX_flat_scaled = scaler.fit_transform(X_flat)\nX_sequences_scaled = X_flat_scaled.reshape(n_samples, n_timesteps, n_features)\n\n# Split data\nX_train, X_val, y_train, y_val = train_test_split(\n    X_sequences_scaled, y_labels, \n    test_size=0.2, \n    random_state=42,\n    stratify=y_labels\n)\n\nprint(f'Training set: {X_train.shape}')\nprint(f'Validation set: {X_val.shape}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Build LSTM model\ndef build_lstm_model(input_shape, num_classes):\n    \"\"\"Build LSTM model for gesture classification\"\"\"\n    \n    model = keras.Sequential([\n        # Input layer\n        layers.Input(shape=input_shape),\n        \n        # LSTM layers\n        layers.LSTM(128, return_sequences=True, dropout=0.2),\n        layers.BatchNormalization(),\n        \n        layers.LSTM(64, return_sequences=True, dropout=0.2),\n        layers.BatchNormalization(),\n        \n        layers.LSTM(32, dropout=0.2),\n        layers.BatchNormalization(),\n        \n        # Dense layers\n        layers.Dense(64, activation='relu'),\n        layers.Dropout(0.3),\n        \n        layers.Dense(32, activation='relu'),\n        layers.Dropout(0.3),\n        \n        # Output layer\n        layers.Dense(num_classes, activation='softmax')\n    ])\n    \n    return model\n\n# Create model\ninput_shape = (X_train.shape[1], X_train.shape[2])\nmodel_lstm = build_lstm_model(input_shape, num_classes)\n\n# Compile model\nmodel_lstm.compile(\n    optimizer=keras.optimizers.Adam(learning_rate=0.001),\n    loss='sparse_categorical_crossentropy',\n    metrics=['accuracy']\n)\n\nmodel_lstm.summary()"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Train LSTM model\nprint('Training LSTM model...')\n\n# Callbacks\nearly_stopping = EarlyStopping(\n    monitor='val_accuracy',\n    patience=10,\n    restore_best_weights=True,\n    mode='max'\n)\n\nreduce_lr = ReduceLROnPlateau(\n    monitor='val_loss',\n    factor=0.5,\n    patience=5,\n    min_lr=0.00001\n)\n\n# Train\nhistory = model_lstm.fit(\n    X_train, y_train,\n    validation_data=(X_val, y_val),\n    epochs=50,\n    batch_size=32,\n    callbacks=[early_stopping, reduce_lr],\n    verbose=1\n)\n\n# Evaluate\nval_loss, val_accuracy = model_lstm.evaluate(X_val, y_val, verbose=0)\nprint(f'\\nLSTM Validation Accuracy: {val_accuracy:.4f}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Feature extraction for traditional ML\ndef extract_features(df_seq):\n    \"\"\"Extract statistical features for traditional ML\"\"\"\n    features = {}\n    \n    # Acceleration features\n    for axis in ['acc_x', 'acc_y', 'acc_z']:\n        if axis in df_seq.columns:\n            features[f'{axis}_mean'] = df_seq[axis].mean()\n            features[f'{axis}_std'] = df_seq[axis].std()\n            features[f'{axis}_max'] = df_seq[axis].max()\n            features[f'{axis}_min'] = df_seq[axis].min()\n    \n    # Rotation features\n    for rot in ['rot_x', 'rot_y', 'rot_z', 'rot_w']:\n        if rot in df_seq.columns:\n            features[f'{rot}_mean'] = df_seq[rot].mean()\n            features[f'{rot}_std'] = df_seq[rot].std()\n    \n    # Temperature features\n    temp_cols = [f'thm_{i}' for i in range(1, 6)]\n    temp_cols = [col for col in temp_cols if col in df_seq.columns]\n    if temp_cols:\n        temp_data = df_seq[temp_cols]\n        features['temp_mean'] = temp_data.mean().mean()\n        features['temp_std'] = temp_data.std().mean()\n    \n    # Magnitude features\n    if all(col in df_seq.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n        acc_mag = np.sqrt(df_seq['acc_x']**2 + df_seq['acc_y']**2 + df_seq['acc_z']**2)\n        features['acc_mag_mean'] = acc_mag.mean()\n        features['acc_mag_std'] = acc_mag.std()\n    \n    features['sequence_length'] = len(df_seq)\n    \n    return features\n\n# Extract features for traditional ML\nprint('Extracting features for traditional ML...')\nX_features = []\ny_features = []\n\nfor seq_id in sequence_ids[:1000]:  # Same subset\n    seq_data = train_df[train_df['sequence_id'] == seq_id]\n    \n    features = extract_features(seq_data)\n    X_features.append(features)\n    \n    gesture = seq_data['gesture'].iloc[0]\n    label = le.transform([gesture])[0]\n    y_features.append(label)\n\nX_features_df = pd.DataFrame(X_features)\ny_features_array = np.array(y_features)\n\n# Split data\nX_train_feat, X_val_feat, y_train_feat, y_val_feat = train_test_split(\n    X_features_df, y_features_array,\n    test_size=0.2,\n    random_state=42,\n    stratify=y_features_array\n)\n\nprint(f'Feature shape: {X_features_df.shape}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Train LightGBM\nprint('Training LightGBM...')\n\nlgb_params = {\n    'objective': 'multiclass',\n    'num_class': num_classes,\n    'metric': 'multi_logloss',\n    'boosting_type': 'gbdt',\n    'num_leaves': 50,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0,\n    'seed': 42\n}\n\ntrain_data = lgb.Dataset(X_train_feat, label=y_train_feat)\nvalid_data = lgb.Dataset(X_val_feat, label=y_val_feat)\n\nmodel_lgb = lgb.train(\n    lgb_params,\n    train_data,\n    valid_sets=[valid_data],\n    num_boost_round=500,\n    callbacks=[lgb.early_stopping(50), lgb.log_evaluation(0)]\n)\n\n# Evaluate\npred_lgb = model_lgb.predict(X_val_feat, num_iteration=model_lgb.best_iteration)\npred_lgb_class = np.argmax(pred_lgb, axis=1)\nlgb_accuracy = np.mean(pred_lgb_class == y_val_feat)\nprint(f'LightGBM Validation Accuracy: {lgb_accuracy:.4f}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Ensemble predictions\nprint('\\n=== Ensemble Results ===')\n\n# LSTM predictions\npred_lstm = model_lstm.predict(X_val, verbose=0)\npred_lstm_class = np.argmax(pred_lstm, axis=1)\nlstm_acc = np.mean(pred_lstm_class == y_val)\n\n# LightGBM predictions (already computed)\n\n# Ensemble\nensemble_weights = [0.6, 0.4]  # LSTM, LightGBM\n\n# Need to align validation sets - using indices for simplicity\n# In production, would properly track sequence IDs\npred_ensemble = ensemble_weights[0] * pred_lstm + ensemble_weights[1] * pred_lgb\npred_ensemble_class = np.argmax(pred_ensemble, axis=1)\nensemble_accuracy = np.mean(pred_ensemble_class == y_val)\n\nprint(f'LSTM Accuracy: {lstm_acc:.4f}')\nprint(f'LightGBM Accuracy: {lgb_accuracy:.4f}')\nprint(f'\\n\ud83c\udfaf Ensemble Accuracy: {ensemble_accuracy:.4f}')\nprint(f'Target 87%: {\"\u2705 ACHIEVED\" if ensemble_accuracy >= 0.87 else \"\u274c NOT YET\"}')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Save models\nprint('Saving models...')\n\n# Save LSTM model\nmodel_lstm.save('lstm_model_v7.h5')\n\n# Save LightGBM model\nwith open('lgb_model_v7.pkl', 'wb') as f:\n    pickle.dump(model_lgb, f)\n\n# Save preprocessing objects\nwith open('scaler_v7.pkl', 'wb') as f:\n    pickle.dump(scaler, f)\n\nwith open('label_encoder_v7.pkl', 'wb') as f:\n    pickle.dump(le, f)\n\nprint('Models saved successfully')"}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": "# Inference function for CMI server\nimport sys\nsys.path.append('/kaggle/input/cmi-detect-behavior-with-sensor-data')\nfrom kaggle_evaluation.cmi_inference_server import CMIInferenceServer\n\ndef predict(sequence, demographics):\n    \"\"\"Prediction function for CMI inference server\"\"\"\n    try:\n        # Convert to DataFrame\n        df_seq = pd.DataFrame(sequence)\n        \n        # Prepare for LSTM\n        seq_data = prepare_sequence_data(df_seq)\n        seq_padded = pad_sequences_custom([seq_data], maxlen=100)\n        \n        # Scale\n        seq_flat = seq_padded.reshape(-1, seq_padded.shape[-1])\n        seq_scaled = scaler.transform(seq_flat)\n        seq_final = seq_scaled.reshape(1, 100, -1)\n        \n        # LSTM prediction\n        pred_lstm = model_lstm.predict(seq_final, verbose=0)\n        \n        # Extract features for LightGBM\n        features = extract_features(df_seq)\n        X_feat = pd.DataFrame([features])\n        \n        # LightGBM prediction\n        pred_lgb = model_lgb.predict(X_feat, num_iteration=model_lgb.best_iteration)\n        \n        # Ensemble\n        pred_ensemble = 0.6 * pred_lstm + 0.4 * pred_lgb\n        \n        # Get predicted class\n        pred_class = np.argmax(pred_ensemble[0])\n        pred_gesture = le.inverse_transform([pred_class])[0]\n        \n        return pred_gesture\n        \n    except Exception as e:\n        print(f'Prediction error: {e}')\n        return 'Text on phone'  # Most common class\n\nprint('Starting CMI Inference Server...')\nserver = CMIInferenceServer(predict)\nserver.serve()\nprint('Inference complete')"}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.0"}}, "nbformat": 4, "nbformat_minor": 4}