{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":242954653,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom tensorflow.keras.models import Sequential, load_model, Model\nfrom tensorflow.keras.layers import (Conv1D, MaxPooling1D, Flatten, Dense, Dropout, BatchNormalization,\n                                   Input, MultiHeadAttention, LayerNormalization, GlobalAveragePooling1D,\n                                   Add, Embedding)\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.optimizers import AdamW\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\nimport tensorflow as tf\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server\nprint(\"Imports loaded\")\n\n# Load the dataset\nprint(\"Loading dataset...\")\ndf = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\nprint(f\"Loaded {len(df)} rows.\")\n\nlabel_encoder = LabelEncoder()\ndf['gesture'] = label_encoder.fit_transform(df['gesture'].astype(str))\n\n# Save class names for inference\nnp.save('gesture_classes.npy', label_encoder.classes_)\n\n# Print class label mapping\nprint(\"Gesture label mapping:\")\nfor idx, label in enumerate(label_encoder.classes_):\n    print(f\"  {idx}: {label}\")\n\nprint(\"Checking for IMU-only sequences...\")\n\ndef check_for_imu_only_seqs():\n    # Identify thermopile and TOF columns\n    thermal_tof_cols = [col for col in df.columns if col.startswith('thm_') or col.startswith('tof_')]\n    \n    # Group by sequence and check if all thm_/tof_ values are null\n    imu_only_flags = df[thermal_tof_cols].isna().groupby(df['sequence_id']).all().all(axis=1)\n    \n    # Report statistics\n    total_sequences = df['sequence_id'].nunique()\n    imu_only_count = imu_only_flags.sum()\n    imu_only_pct = (imu_only_count / total_sequences) * 100\n    \n    print(f\"Total sequences: {total_sequences}\")\n    print(f\"IMU-only sequences (all thm_/tof_ null): {imu_only_count} ({imu_only_pct:.1f}%)\")\n\ncheck_for_imu_only_seqs()\n\nexcluded_cols = {\n    'gesture', 'sequence_type', 'behavior', 'orientation',  # train-only\n    'row_id', 'subject', 'phase',  # metadata\n    'sequence_id', 'sequence_counter'  # identifiers\n}\n\n# Setting this true makes model ignore thermal and tof data\ndrop_thermal_and_tof = True\n\nif drop_thermal_and_tof:\n    thermal_tof_cols = [col for col in df.columns if col.startswith('thm_') or col.startswith('tof_')]\n    excluded_cols.update(thermal_tof_cols)\n    print(f\"Ignoring {len(thermal_tof_cols)} thermopile / time-of-flight columns.\")\n\n# Select numeric feature columns\nfeature_cols = [col for col in df.columns if col not in excluded_cols]\nprint(f\"Using {len(feature_cols)} numeric feature columns for training:\")\nprint(feature_cols)\n\n# Check for NaNs in selected feature columns\nnan_counts = df[feature_cols].isna().sum()\ntotal_nans = nan_counts.sum()\nprint(f\"\\nTotal missing values in feature columns: {total_nans}\")\nif total_nans > 0:\n    print(\"Columns with missing values:\")\n    print(nan_counts[nan_counts > 0])\nelse:\n    print(\"No missing values found in feature columns.\")\n\ndef preprocess_sequence(df_sequence: pd.DataFrame, feature_cols: list) -> np.ndarray:\n    data = df_sequence[feature_cols].copy()\n    data = data.ffill().bfill().fillna(0)\n    scaled = StandardScaler().fit_transform(data)\n    return scaled\n\n# Build sequences\nsequence_ids = df['sequence_id'].unique()\nsequences = df.groupby('sequence_id')\n\nX = []\nseq_lengths = []\n\nprint(\"Building sequences...\")\nfor i, (seq_id, seq) in enumerate(sequences):\n    if i % 500 == 0:\n        print(f\"Processing sequence {i}...\")\n    processed = preprocess_sequence(seq, feature_cols)\n    X.append(processed)\n    seq_lengths.append(processed.shape[0])\n\nmax_len_perentile = 90\n\n# Report sequence length stats\nminlen = min(seq_lengths)\navglen = int(np.mean(seq_lengths))\npad_len_to_use = int(np.percentile(seq_lengths, max_len_perentile))  \nprint(f\"Sequence length stats - Min: {minlen}, Avg: {avglen}, {max_len_perentile}th percentile: {pad_len_to_use}\")\nprint(f\"Padding / truncating all sequences to fixed length {pad_len_to_use}...\")\n\nnp.save(\"sequence_maxlen.npy\", pad_len_to_use)  # Save for inference\n\n# Pad/truncate to fixed length\nX = pad_sequences(X, maxlen=pad_len_to_use, dtype='float32', padding='post', truncating='post')\n\n# Use groupby to get the first gesture per sequence (already integer-encoded)\ny = df.groupby('sequence_id')['gesture'].first().values\n\nprint(\"Integer labels:\", y[:4])\n\n# Convert to one-hot vectors\nnum_classes = len(np.unique(y))\ny = to_categorical(y, num_classes=num_classes)\n\nprint(\"After one-hot encoding:\", y[:4])\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Custom Positional Encoding Layer\nclass PositionalEncoding(tf.keras.layers.Layer):\n    def __init__(self, d_model, max_length=10000, **kwargs):\n        super(PositionalEncoding, self).__init__(**kwargs)\n        self.d_model = d_model\n        self.max_length = max_length\n        \n    def build(self, input_shape):\n        # Create positional encoding matrix\n        position = np.arange(self.max_length)[:, np.newaxis]\n        div_term = np.exp(np.arange(0, self.d_model, 2) * -(np.log(10000.0) / self.d_model))\n        \n        pe = np.zeros((self.max_length, self.d_model))\n        pe[:, 0::2] = np.sin(position * div_term)\n        pe[:, 1::2] = np.cos(position * div_term)\n        \n        self.pe = self.add_weight(\n            name='positional_encoding',\n            shape=(self.max_length, self.d_model),\n            initializer='zeros',\n            trainable=False\n        )\n        self.pe.assign(pe)\n        super(PositionalEncoding, self).build(input_shape)\n    \n    def call(self, inputs):\n        seq_len = tf.shape(inputs)[1]\n        return inputs + self.pe[:seq_len, :]\n    \n    def get_config(self):\n        config = super(PositionalEncoding, self).get_config()\n        config.update({\n            'd_model': self.d_model,\n            'max_length': self.max_length\n        })\n        return config\n\n# Custom Transformer Block\nclass TransformerBlock(tf.keras.layers.Layer):\n    def __init__(self, d_model, num_heads, dff, dropout_rate=0.1, **kwargs):\n        super(TransformerBlock, self).__init__(**kwargs)\n        self.d_model = d_model\n        self.num_heads = num_heads\n        self.dff = dff\n        self.dropout_rate = dropout_rate\n        \n        self.mha = MultiHeadAttention(num_heads=num_heads, key_dim=d_model)\n        self.ffn = tf.keras.Sequential([\n            Dense(dff, activation='relu'),\n            Dense(d_model)\n        ])\n        \n        self.layernorm1 = LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = LayerNormalization(epsilon=1e-6)\n        \n        self.dropout1 = Dropout(dropout_rate)\n        self.dropout2 = Dropout(dropout_rate)\n    \n    def call(self, inputs, training=None, mask=None):\n        # Multi-head self-attention\n        attn_output = self.mha(inputs, inputs, attention_mask=mask, training=training)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        \n        # Feed forward network\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        out2 = self.layernorm2(out1 + ffn_output)\n        \n        return out2\n    \n    def get_config(self):\n        config = super(TransformerBlock, self).get_config()\n        config.update({\n            'd_model': self.d_model,\n            'num_heads': self.num_heads,\n            'dff': self.dff,\n            'dropout_rate': self.dropout_rate\n        })\n        return config\n\n# Build CNN + Self-Attention (Transformer) model\nprint(\"Building CNN + Transformer model...\")\n\ndef create_cnn_transformer_model(input_shape, num_classes):\n    inputs = Input(shape=input_shape)\n    \n    # CNN Feature Extraction Blocks\n    x = Conv1D(64, kernel_size=3, activation='relu', padding='same')(inputs)\n    x = BatchNormalization()(x)\n    x = Conv1D(64, kernel_size=3, activation='relu', padding='same')(x)\n    x = MaxPooling1D(2)(x)\n    x = Dropout(0.3)(x)\n    \n    x = Conv1D(128, kernel_size=5, activation='relu', padding='same')(x)\n    x = BatchNormalization()(x)\n    x = Conv1D(128, kernel_size=5, activation='relu', padding='same')(x)\n    x = MaxPooling1D(2)(x)\n    x = Dropout(0.3)(x)\n    \n    x = Conv1D(256, kernel_size=7, activation='relu', padding='same')(x)\n    x = BatchNormalization()(x)\n    x = Conv1D(256, kernel_size=7, activation='relu', padding='same')(x)\n    x = MaxPooling1D(2)(x)\n    x = Dropout(0.4)(x)\n    \n    # Project to transformer dimension\n    d_model = 256\n    x = Dense(d_model)(x)\n    \n    # Add Positional Encoding\n    x = PositionalEncoding(d_model)(x)\n    \n    # Transformer Blocks (Multi-Head Self-Attention)\n    x = TransformerBlock(d_model=d_model, num_heads=8, dff=512, dropout_rate=0.3)(x)\n    x = TransformerBlock(d_model=d_model, num_heads=8, dff=512, dropout_rate=0.3)(x)\n    x = TransformerBlock(d_model=d_model, num_heads=4, dff=256, dropout_rate=0.3)(x)\n    \n    # Global Average Pooling instead of taking the last token\n    x = GlobalAveragePooling1D()(x)\n    \n    # Classification Head\n    x = Dense(512, activation='relu')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.5)(x)\n    \n    x = Dense(256, activation='relu')(x)\n    x = BatchNormalization()(x)\n    x = Dropout(0.4)(x)\n    \n    x = Dense(128, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    \n    outputs = Dense(num_classes, activation='softmax')(x)\n    \n    model = Model(inputs=inputs, outputs=outputs)\n    return model\n\n# Create the model\nmodel = create_cnn_transformer_model(\n    input_shape=(X_train.shape[1], X_train.shape[2]), \n    num_classes=num_classes\n)\n\n# Compile model with categorical crossentropy loss (for one-hot labels)\nmodel.compile(\n    optimizer=AdamW(learning_rate=0.001, weight_decay=1e-4), \n    loss='categorical_crossentropy', \n    metrics=['accuracy']\n)\nmodel.summary()\n\n# Define callbacks for better training\ncallbacks = [\n    ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=1e-7, verbose=1),\n    EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True, verbose=1)\n]\n\n# Train model using explicitly split validation set (80/20 held out)\nprint(\"Training CNN + Transformer model...\")\nhistory = model.fit(\n    X_train, y_train, \n    epochs=60, \n    batch_size=128, \n    validation_data=(X_val, y_val),\n    callbacks=callbacks,\n    verbose=1\n)\nmodel.save(\"gesture_cnn_transformer_model.h5\")\nprint(\"Training complete.\")\n\nfrom cmi_2025_metric_copy_for_import import CompetitionMetric\n\n# Get predicted labels for the validation set\nprint(\"Predicting on validation set...\")\ny_val_pred_probs = model.predict(X_val, verbose=0)\ny_val_pred = np.argmax(y_val_pred_probs, axis=1)\ny_val_true = np.argmax(y_val, axis=1)\n\n# Map integer labels back to gesture strings\ngesture_classes = np.load(\"gesture_classes.npy\", allow_pickle=True)\nval_pred_labels = pd.Series(y_val_pred).map(lambda i: gesture_classes[i])\nval_true_labels = pd.Series(y_val_true).map(lambda i: gesture_classes[i])\n\n# Build DataFrames for the metric\nval_submission = pd.DataFrame({'gesture': val_pred_labels})\nval_solution = pd.DataFrame({'gesture': val_true_labels})\n\n# Run competition metric\nmetric = CompetitionMetric()\nscore = metric.calculate_hierarchical_f1(val_solution, val_submission)\nprint(f\"Estimated leaderboard (val) score: {score:.4f}\")\n\ndef predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    df_seq = sequence.to_pandas()\n    processed = preprocess_sequence(df_seq, feature_cols)\n    maxlen = int(np.load(\"sequence_maxlen.npy\"))  # ensure consistent shape\n    padded = pad_sequences([processed], maxlen=maxlen, dtype='float32', padding='post', truncating='post')\n    model = load_model(\"gesture_cnn_transformer_model.h5\", \n                      custom_objects={\n                          'PositionalEncoding': PositionalEncoding,\n                          'TransformerBlock': TransformerBlock\n                      })\n    prediction = model.predict(padded, verbose=0)\n    predicted_index = np.argmax(prediction, axis=1)[0]\n    gesture_classes = np.load(\"gesture_classes.npy\", allow_pickle=True)\n    return gesture_classes[predicted_index]\n\n# Launch inference server\ninference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )\n\n# Manual test (only runs outside Kaggle gateway)\nif not os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    print(\"\\nRunning manual test...\")\n    test_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\n    sample_seq_id = test_df['sequence_id'].unique()[0]\n    test_seq = test_df[test_df['sequence_id'] == sample_seq_id]\n    prediction = predict(pl.DataFrame(test_seq), None)\n    print(f\"Manual prediction result for sequence_id {sample_seq_id}: {prediction}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-01T09:17:14.681039Z","iopub.execute_input":"2025-06-01T09:17:14.681315Z","iopub.status.idle":"2025-06-01T09:19:13.647367Z","shell.execute_reply.started":"2025-06-01T09:17:14.681292Z","shell.execute_reply":"2025-06-01T09:19:13.646486Z"}},"outputs":[],"execution_count":null}]}