{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"},{"sourceId":12328761,"sourceType":"datasetVersion","datasetId":7771623},{"sourceId":12362602,"sourceType":"datasetVersion","datasetId":7794493},{"sourceId":242954653,"sourceType":"kernelVersion"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#Basic Imports:\nimport pandas as pd\nimport numpy as np\nimport torch\nimport joblib\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\n#from sklearn.experimental import enable_iterative_imputer\n#from sklearn.impute import IterativeImputer\nimport xgboost as xgb\n\n#Kaggle Scoring Imports:\nimport os\nimport polars as pl\nimport kaggle_evaluation.cmi_inference_server","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T07:05:21.818513Z","iopub.execute_input":"2025-09-09T07:05:21.818797Z","iopub.status.idle":"2025-09-09T07:05:21.823359Z","shell.execute_reply.started":"2025-09-09T07:05:21.818774Z","shell.execute_reply":"2025-09-09T07:05:21.822571Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Part 1 - Setting the training data up","metadata":{}},{"cell_type":"code","source":"#To do:\n# Submit to Kaggle with the correct submission format\n# Rerun grid search to find best imputer params\n# Change model structure & layers + hyperparameters till perfection\n\nFOLDS = 5\nEPOCHS_PER_FOLD = 50 # A fixed number of epochs for simplicity, WILL UPGRADE LATER\nRANDOM_STATE = 69\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f'Using device: {DEVICE}')\n\n#Loading data:\ndf_raw = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\ndemographics_df = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv\")\ndf = pd.merge(df_raw, demographics_df, on='subject', how='left')\n\n# Numerically encoding the target variable:\nle = LabelEncoder()\ndf['gesture_int'] = le.fit_transform(df['gesture'])\nnp.save(\"gesture_classes.npy\", le.classes_)\nprint(\"Numpy class array saved\")\n\n# Separating columns by type for preprocessing and future use in separate model branches:\nimu_cols = [c for c in df_raw.columns if c.startswith('acc_') or c.startswith('rot_')] #movement (speed + rotation)\nthm_cols = [c for c in df_raw.columns if c.startswith('thm_')] #temperature\ntof_cols = [c for c in df_raw.columns if c.startswith('tof_')] #time of flight distance sensors\ndemographics_cols = ['adult_child', 'age', 'sex', 'handedness', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm']\n\n# Adding custom engineered features:\ndf['acc_mag'] = ((df['acc_x'])**2 + (df['acc_y'])**2 + (df['acc_z'])**2)**(1/2)\ndf['temperature'] = df[thm_cols].mean(axis=1)\nnew_tof_features_list = []\nfor i in range(1, 6):\n    tof_df = df[tof_cols[64*(i-1):64*i]].replace(-1.0, np.nan)\n    new_tof_features_list.extend([f'tof_{i}_mean', f'tof_{i}_median', f'tof_{i}_std', f'tof_{i}_max', f'tof_{i}_min'])\n    df[f'tof_{i}_mean'] = tof_df.mean(axis=1, skipna=True)\n    df[f'tof_{i}_median'] = tof_df.median(axis=1, skipna=True)\n    df[f'tof_{i}_std'] = tof_df.std(axis=1, skipna=True)\n    df[f'tof_{i}_max'] = tof_df.max(axis=1, skipna=True)\n    df[f'tof_{i}_min'] = tof_df.min(axis=1, skipna=True)\n\ndf = df.dropna()\n\n\n# # Imputing missing values:\n# features_to_impute = ['rot_w', 'rot_x', 'rot_y', 'rot_z', 'temperature']\n# imputer = IterativeImputer( #Best params found via GridSearchCV, should run this again ...\n#     max_iter=17, tol=0.04, n_nearest_features=3, initial_strategy='mean', random_state=420\n# )\n# df[features_to_impute] = imputer.fit_transform(df[features_to_impute])\n# joblib.dump(imputer, \"imputer.pkl\")\n\n# Normalization of all continuous features (raw and engineered)\ncontinuous_features = ['age', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm', 'acc_x', 'acc_y', 'acc_z', 'acc_mag', 'rot_w', 'rot_x', 'rot_y', 'rot_z', 'temperature'] + thm_cols + new_tof_features_list\nscaler = StandardScaler().fit(df[continuous_features])\ndf[continuous_features] = scaler.transform(df[continuous_features])\njoblib.dump(scaler, \"scaler.pkl\")\n\n\n#1. TWO BRANCH MODEL INPUT - Initializing & padding final IMU+TOF dataset for PyTorch model training:\nX_list, y_list = [], []\nsequence_lengths = df_raw.groupby('sequence_id').size() #Number of rows in each sequence (raw df)\npad_len = int(np.percentile(sequence_lengths, 95)) #99th percentile of sequence lengths for padding\nprint(f\"Padding length set to: {pad_len}\")\n\nseq_gp = df.groupby('sequence_id') #Iterable groupby object, of sequences in the edited dataframe\nimu_and_tof_features = imu_cols + tof_cols\n\nfor seq_id, seq in seq_gp: #Iterates through each sequence group of rows\n    seq_data = seq[imu_and_tof_features].values #Imu and tof features of each sequence\n    #Padded sequence:\n    padded_seq = np.zeros((pad_len, len(imu_and_tof_features))) #Very low weight is assigned to the parts that remain 0\n    padded_seq[:min(len(seq_data), pad_len)] = seq_data[:min(len(seq_data), pad_len)]\n    \n    X_list.append(padded_seq) # Store the whole sequence for processing\n    y_list.append(seq['gesture_int'].iloc[0]) #1 y label per sequence\n\ny_labels = np.array(y_list) #NumPy array of y labels, 1 for each sequence\nX_padded = np.array(X_list) #NumPy array of padded sequences\n\n\n#2. XGBOOST ADDITIONAL INPUT: Preparing demographics and engineered tof features for XGBoost model training\ndemographics_meta_df = df.groupby('sequence_id').first().reset_index() #First row of each sequence\ndemo_cols = ['age', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm']\n\ndemographics_meta_df = pd.get_dummies(demographics_meta_df, columns=['adult_child', 'sex', 'handedness'], drop_first=True) #One Hot Encoding the categorical demographic columns\ndemo_cols.extend([c for c in demographics_meta_df.columns if 'adult_child' in c or 'sex' in c or 'handedness' in c]) #Adds all the new columns created by one-hot-encoding to the demographic columns.\n#^^^Columns will be formatted like: adult_child_0, adult_child_1, sex_0, sex_1, handedness_0, handedness_1 (but the 0s will be dropped to reduce multicollinearity)\ndemographics_meta = demographics_meta_df[demo_cols].values #One NumPy array for each sequence, with values for all demographic columns.\n\nengineered_tof_meta_df = df.groupby('sequence_id').first().reset_index() #The summary stats\nengineered_tof_meta = engineered_tof_meta_df[new_tof_features_list].values\n\n\n# Summary of all prepared data:\nprint(\"STEP ONE: Two Branch PyTorch Model Input\")\nprint('Training dataset gesture labels labels:')\nprint(y_labels)\nprint('Padded IMU and TOF sensor training data for every sequence:')\nprint(X_padded) #Shape: (pad_len) # of rows and (imu+tof features) # of columns, for all sequences.\n\nprint(\"STEP TWO: Additional XGBoost Model Input (to be combined with output of the PyTorch model)\")\nprint('Tof features:')\nprint(new_tof_features_list) #Summary stats for each of the 5 tof sensors\nprint('Engineered_tof_meta:')\nprint(engineered_tof_meta) #One NumPy array of summary statistics for each sequence.\nprint('Demographics columns:')\nprint(demo_cols) \nprint('Demographics meta:') \nprint(demographics_meta) #One NumPy array for each sequence, with values for all demographic columns.","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T07:33:36.231122Z","iopub.execute_input":"2025-09-09T07:33:36.231744Z","iopub.status.idle":"2025-09-09T07:34:22.223964Z","shell.execute_reply.started":"2025-09-09T07:33:36.231721Z","shell.execute_reply":"2025-09-09T07:34:22.223164Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Part 2 - Training and Validating","metadata":{}},{"cell_type":"code","source":"#Basic Imports:\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold\nimport joblib\nimport xgboost as xgb\nfrom sklearn.preprocessing import LabelEncoder\n\n\n#DEFINING MODEL AND TRAINING FUNCTION:\nclass TwoBranchModel(nn.Module):\n    \"\"\"\n    Inherits from nn.Module, base class for all neural network modules in PyTorch (abstracts the __call__ method)\n    Two branch refers to imu and tof branches. Processing is done separately for each branch, then combined with attention mechanism.\n    \n    ***the layers are callable objects due to inheriting from nn.Module (__call__ method is being called)***\n    ***how linear layer works: Output=Input⋅Weights+Bias --> (Input:(batch size, in_features), Output:(batch size, out_features)). So, weights are (in_features, out_features) matrix.***\n    \"\"\"\n    def __init__(self, imu_dim, tof_dim, n_classes, pad_len, embedding_dim):\n        super(TwoBranchModel, self).__init__() #Calls constructor of the parent class (nn.Module) - requirement for all PyTorch nn models\n        \n        self.imu_dim = imu_dim #num of imu features\n        self.tof_dim = tof_dim #num of tof features\n        self.n_classes = n_classes #num of gesture classes\n        self.pad_len = pad_len #Padding length for sequences - 99th percentile of the number of rows in each sequence, to standardize it.\n        \n        # Defining the two branches with simple linear layers\n        # (UPGRAD ALL LAYERS BELOW TO: LSTMs or Conv1D layers, or others)\n        self.imu_branch = nn.Linear(self.pad_len * self.imu_dim, 64) #Params: (in_features, out_features)\n        self.tof_branch = nn.Linear(self.pad_len * self.tof_dim, 64)\n        #^^^Input size: (batch size, sequence length * num features), output size is: (batch size, 64).\n        \n        # Attention Layer, which learns to weight the outputs of the two branches\n        self.attention_weights = nn.Linear(128, 2) #Inputs: Weights for IMU and TOF, combined (64+64), outputs 2 attention scores.\n        \n        # Final dense layers to produce embedding and logits\n        self.dense1 = nn.Linear(128, embedding_dim) #Dense layer taking in a scaled attention output. Input size is 128 from imu_branch_out and tof_branch_out and projects this data to 256-dim.\n        self.dense2 = nn.Linear(embedding_dim, embedding_dim) #Embedding layer. Takes 256-D output, fine-tunes weights, and OUTPUTS a final 256-D vector (EMBEDDING) of input data.\n        #*Embedding = most important features for each gesture. (256-D vector, one for each gesture)*\n        \n        self.classifier = nn.Linear(embedding_dim, self.n_classes) #Final classification layer. Takes 256-D embedding and outputs logits (confidence or probability) for each class.\n\n    def forward(self, x):\n        \"\"\"\n        Handles data flow through network layers\n        \"\"\"\n        \n        #Splitting input tensor x into the two branches:\n        imu_data = x[:, :, :self.imu_dim] \n        tof_data = x[:, :, self.imu_dim:]\n        \n        # Calling the first branch layers to define and process the imu and tof data separately:\n        imu_branch_out = self.imu_branch(imu_data.reshape(x.size(0), -1)) #Feeds imu data into imu branch layer, flattening from 3D-->2D\n        tof_branch_out = self.tof_branch(tof_data.reshape(x.size(0), -1)) #Feeds tof data into tof branch layer, flattening from 3D-->2D\n        #^^^They become size (batch_size, 64)\n        \n        #Concatenate the outputs and pass through attention layer:\n        combined_out = torch.cat([imu_branch_out, tof_branch_out], dim=1)\n        attention_scores = torch.softmax(self.attention_weights(combined_out), dim=1) #This represents the importance scores of each of the 2 branches. Softmax makes scores sum to 1.\n        attention_out = combined_out * attention_scores.repeat_interleave(64, dim=1) #This is the attention-weighted output. Branch outputs are multiplied by their attention scores.\n\n        #Pass through the dense layers to get embedding and logits:\n        hidden = self.dense1(attention_out) #Transforms from 128--256-D for more complex features\n        embedding = self.dense2(hidden) #Fine-tunes weights again to produce final 256-D embedding\n        logits = self.classifier(embedding) #Gets class scores from the embedding\n        \n        return logits, embedding\n        \"\"\"logits = prediction scores, emmbedding = vector of data after parameters transform it (learned combination of all input features)\"\"\"\n\ndef train_and_predict_pytorch(X_train, y_train, X_val, y_val, imu_dim, tof_dim, n_classes, pad_len, device, embedding_dim):\n    \"\"\"\n    Simple training function for a single CV fold\n    \"\"\"\n    \n    #Loading in the datasets:\n    train_dataset = TensorDataset(torch.from_numpy(X_train).float(), torch.from_numpy(y_train).long())\n    train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\n    val_dataset = TensorDataset(torch.from_numpy(X_val).float(), torch.from_numpy(y_val).long())\n    val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)\n\n    #Initializing model, optimizer, and loss function:\n    model = TwoBranchModel(imu_dim, tof_dim, n_classes, pad_len, embedding_dim).to(device) #Instantiates 2-branch neural network model\n    optimizer = optim.Adam(model.parameters(), lr=0.001) #Implement ReducedLROnPlateau later\n    criterion = nn.CrossEntropyLoss() #Experiment\n\n    #Training epoch:\n    for epoch in range(EPOCHS_PER_FOLD):\n        print(f\"Epoch {epoch+1}/{EPOCHS_PER_FOLD}\")\n        model.train() #Sets model to training mode\n        for X, y in train_loader: #Iterates through the batches of loaded training data (split into batches of 32 sequences)\n            X, y = X.to(device), y.to(device) #Moves data to GPU if available for faster processing\n            optimizer.zero_grad() #Resets all gradients to 0\n            logits, _ = model(X) #Calls base class __call__ method, which calls forward(), alongside managing training state and setting up internal hooks\n            loss = criterion(logits, y) #Gets the loss\n            loss.backward() #Backward pass / performing backpropagation algorithm - gradients of loss w respect to model params are calculated\n            optimizer.step() #Update model params using calculated gradients. (where learning occurs)\n    \n    # Get predictions and embeddings on the validation set:\n    model.eval() #Sets model to evaluation mode\n    val_preds = []\n    val_embeddings = []\n    with torch.no_grad(): #Gradients are not calculated\n        for X, _ in val_loader: #Iterates through validation data\n            X = X.to(device)\n            logits, embedding = model(X) #forward pass to get logits\n            val_preds.append(torch.softmax(logits, dim=1).cpu().numpy()) #Val probabilities are gotten thru softmax\n            val_embeddings.append(embedding.cpu().numpy())\n    \n    return np.vstack(val_preds), np.vstack(val_embeddings), model #Stores predictions and embeddings and model for later use.\n\n\n# PERFORMING THE TRAINING FUNCTION WITH K-FOLD CV:\n\nEMBEDDING_DIM = 256 #Size of the embedding vector from the NN - CONFIGURE HERE\n\n#Initializing storage of oof preds and embeddings for each fold:\n\"\"\"\noof = out of fold predictions\noof data is the validation data for each fold (outside the training fold)\n\"\"\"\noof_preds_nn = np.zeros((len(y_labels), len(le.classes_))) #Stores oof (validation) predictions for Two-Branch NN. num of individual gesture sequences (y-labels) times num of possible classes (le.classes_)\noof_preds_xgb = np.zeros((len(y_labels), len(le.classes_))) #Stores oof (validation) predictions for XGBoost model. num of individual gesture sequences (y-labels) times num of possible classes (le.classes_)\noof_features_nn = np.zeros((len(y_labels), EMBEDDING_DIM)) #Stores embeddings. num of sequences * embedding size\ntrained_nn_models = [] #Stores a PyTorch model for each fold\n\nskf = StratifiedKFold(n_splits=FOLDS, shuffle=True, random_state=RANDOM_STATE) #Stratified K-Folds CV ensures equal proportion in each gesture class\n\nfor fold, (train_idx, val_idx) in enumerate(skf.split(X_padded, y_labels)):\n    \"\"\"\n    skf.split splits dataset into stratified train-test indices\n    rain_idx and val_idx are arrays of integer indices.\n    \"\"\"\n    print(f\"--- Fold {fold+1}/{FOLDS} ---\")\n    \n    # Initalizing fold data:\n    X_train_ts, X_val_ts = X_padded[train_idx], X_padded[val_idx]\n    y_train_ts, y_val_ts = y_labels[train_idx], y_labels[val_idx]\n    \n    # Getting additional XGBoost features for the fold:\n    demographics_train, demographics_val = demographics_meta[train_idx], demographics_meta[val_idx]\n    engineered_tof_train, engineered_tof_val = engineered_tof_meta[train_idx], engineered_tof_meta[val_idx]\n\n    # 1. Train PyTorch Model for fold:\n    print(f\"pad_len: {pad_len}\")\n    nn_preds_fold, nn_embeddings_fold, nn_model = train_and_predict_pytorch(\n        X_train_ts, y_train_ts, X_val_ts, y_val_ts,\n        len(imu_cols), len(tof_cols), len(le.classes_),\n        pad_len, DEVICE, embedding_dim=EMBEDDING_DIM #unexpected keyword argument 'embedding_dim'\n    ) \n    oof_preds_nn[val_idx] = nn_preds_fold #predictions for the validation data\n    oof_features_nn[val_idx] = nn_embeddings_fold #embedding for the validation data\n    trained_nn_models.append(nn_model) #Trained model is saved to the list\n    \n    # 2. Train XGBoost Model on PyTorch output + additional features:\n    xgb_train_features = np.concatenate([oof_features_nn[train_idx], demographics_train, engineered_tof_train], axis=1)\n    xgb_val_features = np.concatenate([nn_embeddings_fold, demographics_val, engineered_tof_val], axis=1)\n    print(\"(TRAIN) nn_embeddings_fold shape:\")\n    print(nn_embeddings_fold.shape)\n    print(\"(TRAIN) Demographics_val shape:\")\n    print(demographics_val.shape)\n    print(demographics_val)\n    print(\"(TRAIN) Engineered_tof_val shape:\")\n    print(engineered_tof_val.shape)\n    \n    xgb_clf = xgb.XGBClassifier(n_estimators=100, eval_metric='mlogloss', tree_method='gpu_hist' if DEVICE == 'cuda' else 'auto')\n    xgb_clf.fit(xgb_train_features, y_train_ts)\n    \n    oof_preds_xgb[val_idx] = xgb_clf.predict_proba(xgb_val_features) #saves oof predictions from xgboost to oof_preds_xgb array\n    joblib.dump(xgb_clf, f'xgb_fold_{fold}.pkl') #xgb model for this fold is saved to disk\n\n# 3. Train the Combined Meta-Model\n\"\"\"\nMeta-model learns the strengths and weaknesses of the two base models based on the confidence of their predictions (based on probability spread), \non a bunch of out of fold predictions which makes the predictions representative of predictions on unseen data as opposed to just memorizing train data, \nto make a final, more accurate prediction. Consider the meta-model a teacher taking the strengths of its two students to make a final answer.\n\"\"\"\n#Creating final dataset of past predictions to perform stacking:\nmeta_features = np.concatenate([oof_preds_nn, oof_preds_xgb], axis=1) #Stacks ALL THE oof predictions from pytorch and xgboost to create final dataset of meta-features for meta-model\n\n#Create and train final meta-model:\nmeta_model = xgb.XGBClassifier(n_estimators=50, eval_metric='mlogloss', tree_method='gpu_hist' if DEVICE == 'cuda' else 'auto')\nmeta_model.fit(meta_features, y_labels)\n\n# Saving final meta-model & trained NN models to disk, for inference:\njoblib.dump(meta_model, \"meta_model.pkl\")\nfor i, model in enumerate(trained_nn_models):\n    print(\"Model fold path:\")\n    torch.save(model.state_dict(), f\"pytorch_fold_{i}.pth\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T07:09:40.317265Z","iopub.execute_input":"2025-09-09T07:09:40.317870Z","iopub.status.idle":"2025-09-09T07:09:55.520817Z","shell.execute_reply.started":"2025-09-09T07:09:40.317847Z","shell.execute_reply":"2025-09-09T07:09:55.519800Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Part 3 - Predicting on Test Data","metadata":{}},{"cell_type":"code","source":"def predict(test_df, test_demographics_df):\n    \"\"\"\n    All models have been saved, now constituting a saved stacking ensemble.\n    Performing inference on test data using that ensemble.\n    \n    .pth is a PyTorch-specific file convention for model weights --> deserialized/loaded through torch.load()\n    .pkl file is a byte stream file that represents a serialized Python object --> loaded through joblib.load()\n    \"\"\"\n    #Loading in the test data:\n    print(\"Loading test data...\")\n    df_raw = test_df.to_pandas()\n    demographics_df = test_demographics_df.to_pandas()\n    df = pd.merge(df_raw, demographics_df, on='subject', how='left')\n    \n    #Loading in the saved models and preprocessors:\n    try:\n        scaler = joblib.load(\"scaler.pkl\")\n        meta_model = joblib.load(\"meta_model.pkl\")\n        #imputer = joblib.load(\"imputer.pkl\")\n        le = LabelEncoder()\n        le.classes_ = np.load(\"gesture_classes.npy\", allow_pickle=True)\n    except FileNotFoundError as e:\n        print(f\"Error: Missing saved model or preprocessor file. Please ensure all files are in the same directory.\")\n        raise e\n\n    #Preprocessing the test data the same way as with the training data:\n    print(\"Preprocessing test data...\")\n    imu_cols = [c for c in df_raw.columns if c.startswith('acc_') or c.startswith('rot_')]\n    thm_cols = [c for c in df_raw.columns if c.startswith('thm_')]\n    tof_cols = [c for c in df_raw.columns if c.startswith('tof_')]\n    \n    df['acc_mag'] = ((df['acc_x'])**2 + (df['acc_y'])**2 + (df['acc_z'])**2)**(1/2)\n    df['temperature'] = df[thm_cols].mean(axis=1)\n\n    new_tof_features_list = []\n    for i in range(1, 6):\n        tof_df = df[tof_cols[64*(i-1):64*i]].replace(-1.0, np.nan)\n        new_tof_features_list.extend([f'tof_{i}_mean', f'tof_{i}_median', f'tof_{i}_std', f'tof_{i}_max', f'tof_{i}_min'])\n        df[f'tof_{i}_mean'] = tof_df.mean(axis=1, skipna=True)\n        df[f'tof_{i}_median'] = tof_df.median(axis=1, skipna=True)\n        df[f'tof_{i}_std'] = tof_df.std(axis=1, skipna=True)\n        df[f'tof_{i}_max'] = tof_df.max(axis=1, skipna=True)\n        df[f'tof_{i}_min'] = tof_df.min(axis=1, skipna=True)\n\n    df = df.dropna()\n    \n    # features_to_impute = ['rot_w', 'rot_x', 'rot_y', 'rot_z', 'temperature']\n    # df[features_to_impute] = imputer.transform(df[features_to_impute])\n\n    continuous_features = ['age', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm', 'acc_x', 'acc_y', 'acc_z', 'acc_mag', 'rot_w', 'rot_x', 'rot_y', 'rot_z', 'temperature'] + thm_cols + new_tof_features_list\n    df[continuous_features] = scaler.transform(df[continuous_features])\n    \n    \n    # Preparing the data for PyTorch and XGBoost in the same way as the training data:\n    print(f\"pad_len: {pad_len}\")\n    imu_and_tof_features = imu_cols + tof_cols\n    \n    X_list = []\n    seq_gp = df.groupby('sequence_id')\n    for seq_id, seq in seq_gp:\n        seq_data = seq[imu_and_tof_features].values\n        padded_seq = np.zeros((pad_len, len(imu_and_tof_features)))\n        padded_seq[:min(len(seq_data), pad_len)] = seq_data[:min(len(seq_data), pad_len)]\n        X_list.append(padded_seq)\n\n    X_padded = np.array(X_list)\n\n    demographics_meta_df = df.iloc[0:1] #Just getting the first value instead of grouping by sequence id and getting the first, as the kaggle_inference calls predict PER sequence.\n    #demographics_meta_df = df.groupby('sequence_id').first().reset_index()\n    \n    #Getting dummy variables for the categorical variables (need to use CategoricalDType to avoid categories that don't appear getting skipped):\n    demo_categorical_cols = ['adult_child', 'sex', 'handedness']\n    demo_categorical_df = pd.DataFrame({\n        'adult_child': pd.Series(demographics_meta_df['adult_child'], dtype=pd.CategoricalDtype(categories=[0,1])),\n        'sex': pd.Series(demographics_meta_df['sex'], dtype=pd.CategoricalDtype(categories=[0,1])),\n        'handedness': pd.Series(demographics_meta_df['handedness'], dtype=pd.CategoricalDtype(categories=[0,1]))\n    })\n    demographics_categorical_dummies_df = pd.get_dummies(demo_categorical_df, demo_categorical_cols, drop_first=True)\n    \n    #Making final demographics data:\n    demographics_meta_df = pd.concat([demographics_meta_df, demographics_categorical_dummies_df], axis=1)\n    demo_cols = ['age', 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm', 'adult_child_1', 'sex_1', 'handedness_1']\n    demographics_meta = demographics_meta_df[demo_cols].values\n    \n    engineered_tof_meta_df = df.groupby('sequence_id').first().reset_index()\n    engineered_tof_meta = engineered_tof_meta_df[new_tof_features_list].values\n    \n    \n    #Generating meta-features for the final meta-model using the base models:\n    print(\"Generating meta-features with the two base models...\")\n    \n    # 1. PyTorch Meta-Features\n    nn_preds_list = []\n    nn_embeddings_list = []\n    for i in range(FOLDS):\n        model = TwoBranchModel(len(imu_cols), len(tof_cols), len(le.classes_), pad_len, embedding_dim=EMBEDDING_DIM).to(DEVICE)\n        pytorch_fold_weights = torch.load(f\"pytorch_fold_{i}.pth\", map_location=DEVICE) #.pth file is model's weights.\n        \n        model.load_state_dict(pytorch_fold_weights)\n        #^^torch.load() returns a state_dict, where keys are the weights and biases of each model layer (dense1.bias, imu_branch.weight), and values are their values\n        #^^^model.load_state_dict() comes from nn.Module - takes in a state_dict and copies the params into the model layers\n        model.eval() #sets model to evaluation mode.\n        with torch.no_grad(): #no gradients are calculated during code as they are not needed, saving time\n            logits, embedding = model(torch.from_numpy(X_padded).float().to(DEVICE)) #Getting raw pred scores and 256-D representation of each gesture\n            nn_preds_list.append(torch.softmax(logits, dim=1).cpu().numpy()) #logits as a prob distribution (moved back to cpu and converted to numpy) are added to nn_preds_list. list of arrays of lists, where each outer array is a fold and each inner list is prob distribution for the sequence.\n            nn_embeddings_list.append(embedding.cpu().numpy()) #Embedding moved to cpu and converted to numpy are added\n    \n    avg_nn_preds = np.mean(nn_preds_list, axis=0) #aligns all five outer arrays and averages value by position. (each position in each inner sequence list is averaged, across all five folds.)\n    avg_nn_embeddings = np.mean(nn_embeddings_list, axis=0) #average learned features are averaged across all five folds\n    \n    # 2. XGBoost Meta-Features\n    xgb_preds_list = []\n    for i in range(FOLDS):\n        xgb_model = joblib.load(f\"xgb_fold_{i}.pkl\")\n        xgb_val_features = np.concatenate([avg_nn_embeddings, demographics_meta, engineered_tof_meta], axis=1)\n        print(\"(TEST) nn_embeddings_fold shape:\")\n        print(avg_nn_embeddings.shape)\n        print(\"(TEST) Demographics_meta shape:\")\n        print(demographics_meta.shape)\n        print(demo_cols)\n        print(demographics_meta)\n        print(\"(TEST) Engineered_tof_meta shape:\")\n        print(engineered_tof_meta.shape)\n        xgb_preds_list.append(xgb_model.predict_proba(xgb_val_features))\n\n\n    avg_xgb_preds = np.mean(xgb_preds_list, axis=0)\n\n    # 3. Final Meta-Model Predictions\n    print(\"Making final predictions with the meta-model...\")\n    meta_features = np.concatenate([avg_nn_preds, avg_xgb_preds], axis=1)\n    final_predictions = meta_model.predict(meta_features)\n    \n    # Convert integer predictions back to gesture names\n    predicted_gestures = le.inverse_transform(final_predictions)\n    \n    return predicted_gestures[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T07:27:54.366997Z","iopub.execute_input":"2025-09-09T07:27:54.367826Z","iopub.status.idle":"2025-09-09T07:27:54.384492Z","shell.execute_reply.started":"2025-09-09T07:27:54.367802Z","shell.execute_reply":"2025-09-09T07:27:54.383741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-09T07:27:57.211125Z","iopub.execute_input":"2025-09-09T07:27:57.211391Z","iopub.status.idle":"2025-09-09T07:27:58.688725Z","shell.execute_reply.started":"2025-09-09T07:27:57.211372Z","shell.execute_reply":"2025-09-09T07:27:58.688035Z"}},"outputs":[],"execution_count":null}]}