{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport math\n\nimport pandas as pd\nimport polars as pl\nimport numpy as np\n\nimport kaggle_evaluation.cmi_inference_server\n\nimport sklearn\nfrom sklearn.decomposition import PCA\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\n\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pad_sequence\nimport torchmetrics\nimport torch.optim as optim\n\nMAX_SEQ_LEN = 210\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:03.992621Z","iopub.execute_input":"2025-06-28T10:41:03.993199Z","iopub.status.idle":"2025-06-28T10:41:18.219843Z","shell.execute_reply.started":"2025-06-28T10:41:03.993176Z","shell.execute_reply":"2025-06-28T10:41:18.219133Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The evaluation API requires that you set up a server which will respond to inference requests. We have already defined the server; you just need write the predict function. When we evaluate your submission on the hidden test set the client defined in the gateway will run in a different container with direct access to the hidden test set and hand off the one sequence at a time.\n\nYour code will always have access to the published copies of the files.","metadata":{"execution":{"iopub.status.busy":"2025-05-29T22:29:18.955791Z","iopub.execute_input":"2025-05-29T22:29:18.956193Z","iopub.status.idle":"2025-05-29T22:29:18.965179Z","shell.execute_reply.started":"2025-05-29T22:29:18.956169Z","shell.execute_reply":"2025-05-29T22:29:18.963797Z"}}},{"cell_type":"code","source":"def seed_everything(seed):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    random.seed(seed)\n    np.random.seed(seed)\nseed_everything(seed=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:18.221117Z","iopub.execute_input":"2025-06-28T10:41:18.221468Z","iopub.status.idle":"2025-06-28T10:41:18.225387Z","shell.execute_reply.started":"2025-06-28T10:41:18.22145Z","shell.execute_reply":"2025-06-28T10:41:18.224701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntest = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\ntrain_demographics = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\ntest_demographics = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv')\ntrain.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:18.226232Z","iopub.execute_input":"2025-06-28T10:41:18.226732Z","iopub.status.idle":"2025-06-28T10:41:47.124433Z","shell.execute_reply.started":"2025-06-28T10:41:18.226705Z","shell.execute_reply":"2025-06-28T10:41:47.12367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train['sequence_id'].value_counts().plot(kind='hist')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.126091Z","iopub.execute_input":"2025-06-28T10:41:47.126298Z","iopub.status.idle":"2025-06-28T10:41:47.129742Z","shell.execute_reply.started":"2025-06-28T10:41:47.126282Z","shell.execute_reply":"2025-06-28T10:41:47.129036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# low, high = train['sequence_id'].value_counts().quantile([0.01, 0.99])\n# low, high","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.130378Z","iopub.execute_input":"2025-06-28T10:41:47.130593Z","iopub.status.idle":"2025-06-28T10:41:47.145057Z","shell.execute_reply.started":"2025-06-28T10:41:47.130573Z","shell.execute_reply":"2025-06-28T10:41:47.144365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.145837Z","iopub.execute_input":"2025-06-28T10:41:47.146035Z","iopub.status.idle":"2025-06-28T10:41:47.160215Z","shell.execute_reply.started":"2025-06-28T10:41:47.14602Z","shell.execute_reply":"2025-06-28T10:41:47.159502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# grouped_gestures = train.groupby('sequence_type')['gesture'].apply(lambda x: list(set(x))).reset_index()\n# grouped_gestures","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.16084Z","iopub.execute_input":"2025-06-28T10:41:47.161109Z","iopub.status.idle":"2025-06-28T10:41:47.175342Z","shell.execute_reply.started":"2025-06-28T10:41:47.161092Z","shell.execute_reply":"2025-06-28T10:41:47.174689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# exploded_df = grouped_gestures.explode('gesture').reset_index(drop=True)\n# exploded_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.176003Z","iopub.execute_input":"2025-06-28T10:41:47.17627Z","iopub.status.idle":"2025-06-28T10:41:47.192377Z","shell.execute_reply.started":"2025-06-28T10:41:47.176252Z","shell.execute_reply":"2025-06-28T10:41:47.191679Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tof_cols = [f'tof_{s}_v{i}' for s in range(1, 6) for i in range(64)]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.193002Z","iopub.execute_input":"2025-06-28T10:41:47.193229Z","iopub.status.idle":"2025-06-28T10:41:47.20878Z","shell.execute_reply.started":"2025-06-28T10:41:47.193214Z","shell.execute_reply":"2025-06-28T10:41:47.208147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop(columns=['sequence_type', 'orientation', 'row_id', 'sequence_counter', 'behavior', 'phase'], inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.211192Z","iopub.execute_input":"2025-06-28T10:41:47.211712Z","iopub.status.idle":"2025-06-28T10:41:47.675982Z","shell.execute_reply.started":"2025-06-28T10:41:47.211693Z","shell.execute_reply":"2025-06-28T10:41:47.675166Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train.groupby('sequence_id')[train.columns.drop('sequence_id')].agg(list).reset_index()\n# y_train = train['gesture'].apply(lambda x: x[0])\n# X_train = train.drop(columns=['gesture'])\n# X_train['subject'] = X_train['subject'].apply(lambda x: x[0])\ntrain['subject'] = train['subject'].apply(lambda x: x[0])\ntrain['gesture'] = train['gesture'].apply(lambda x: x[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:41:47.676806Z","iopub.execute_input":"2025-06-28T10:41:47.67711Z","iopub.status.idle":"2025-06-28T10:42:52.62405Z","shell.execute_reply.started":"2025-06-28T10:41:47.677087Z","shell.execute_reply":"2025-06-28T10:42:52.623458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train_demographics.merge(train, on='subject', how='right')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.624805Z","iopub.execute_input":"2025-06-28T10:42:52.625045Z","iopub.status.idle":"2025-06-28T10:42:52.813055Z","shell.execute_reply.started":"2025-06-28T10:42:52.625027Z","shell.execute_reply":"2025-06-28T10:42:52.812248Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def tensorising(df, dynamic_cols, max_seq_len=210):\n    \"\"\"\n    Converts dynamic sequence data in a DataFrame into padded tensors and captures sequence lengths.\n\n    Args:\n        df (pd.DataFrame): Input DataFrame with sequence data.\n        dynamic_cols (list of str): List of column names containing dynamic sequence data (lists or arrays).\n        max_seq_len (int): Maximum sequence length for truncation and padding.\n\n    Returns:\n        sequence_tensor (torch.Tensor): Batched padded sequence tensor of shape (B, max_seq_len, num_dynamic_cols).\n        seq_lengths (torch.Tensor): Tensor of original (capped) sequence lengths, shape (B,).\n    \"\"\"\n    sequence_tensors = []\n    seq_lengths = []\n\n    for _, row in df.iterrows():\n        cleaned_dynamic_cols = []\n        for col in dynamic_cols:\n            cleaned_col = [0.0 if pd.isna(x) or x == -1 else x for x in row[col]]\n            # cleaned_col = [0.0 if pd.isna(x) else x for x in row[col]]\n            cleaned_dynamic_cols.append(cleaned_col)\n\n        seq = list(zip(*cleaned_dynamic_cols))  # shape: (seq_len, num_dynamic_cols)\n        seq_tensor = torch.tensor(seq, dtype=torch.float32)\n\n        # Truncate\n        if len(seq_tensor) > max_seq_len:\n            seq_tensor = seq_tensor[:max_seq_len]\n\n        # Record actual (capped) sequence length\n        seq_len = min(len(seq_tensor), max_seq_len)\n\n        # Pad if necessary\n        if len(seq_tensor) < max_seq_len:\n            pad_len = max_seq_len - len(seq_tensor)\n            pad_tensor = torch.zeros((pad_len, seq_tensor.shape[1]))\n            seq_tensor = torch.cat([seq_tensor, pad_tensor], dim=0)\n\n        sequence_tensors.append(seq_tensor)\n        seq_lengths.append(seq_len)\n\n    # Convert to final tensors\n    sequence_tensor = torch.stack(sequence_tensors)                # Shape: (B, max_seq_len, D)\n    seq_lengths_tensor = torch.tensor(seq_lengths, dtype=torch.long)\n\n    return sequence_tensor, seq_lengths_tensor","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.81401Z","iopub.execute_input":"2025-06-28T10:42:52.814246Z","iopub.status.idle":"2025-06-28T10:42:52.821087Z","shell.execute_reply.started":"2025-06-28T10:42:52.814229Z","shell.execute_reply":"2025-06-28T10:42:52.820315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocess_dataframe(df, dynamic_cols, static_cols, scaler=None, is_train=True, max_seq_len=210, flatten_dynamic=True):\n    \"\"\"\n    Preprocesses a dataframe into tensors for static and dynamic features.\n\n    Args:\n        df (pd.DataFrame): Input data.\n        dynamic_cols (list): Dynamic sequence columns.\n        static_cols (list): Static feature columns.\n        scaler (StandardScaler): Scaler for static features.\n        is_train (bool): Whether in training mode (fit scaler).\n        max_seq_len (int): Max sequence length.\n        flatten_dynamic (bool): Whether to flatten dynamic sequences for CNN.\n\n    Returns:\n        static_tensors (torch.Tensor): Shape (B, static_dim)\n        dynamic_tensors (torch.Tensor): Shape (B, D*max_seq_len) or (B, max_seq_len, D)\n        seq_lengths (torch.Tensor): Shape (B,)\n        scaler (StandardScaler): Fitted or passed scaler\n    \"\"\"\n    # ----- Static Features -----\n    if is_train:\n        scaler = StandardScaler()\n        static_data = scaler.fit_transform(df[static_cols])\n    else:\n        static_data = scaler.transform(df[static_cols])\n    \n    static_tensors = torch.tensor(static_data, dtype=torch.float32)\n\n    # ----- Dynamic Features -----\n    dynamic_tensor, seq_lengths = tensorising(df, dynamic_cols, max_seq_len=max_seq_len)\n\n    if flatten_dynamic:\n        # For CNN input: (B, D * max_seq_len)\n        dynamic_tensor = dynamic_tensor.view(dynamic_tensor.size(0), -1)\n\n    return static_tensors, dynamic_tensor, seq_lengths, scaler\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.821882Z","iopub.execute_input":"2025-06-28T10:42:52.822173Z","iopub.status.idle":"2025-06-28T10:42:52.843706Z","shell.execute_reply.started":"2025-06-28T10:42:52.822156Z","shell.execute_reply":"2025-06-28T10:42:52.843011Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class HybridDataset(Dataset):\n    def __init__(self, static_data, sequence_data, labels=None, seq_lengths=None):\n        self.static_data = static_data\n        self.sequence_data = sequence_data\n        self.labels = labels\n        self.seq_lengths = seq_lengths  # Optional — pass if needed for RNNs\n\n    def __len__(self):\n        return len(self.static_data)\n\n    def __getitem__(self, idx):\n        sample = {\n            'static': self.static_data[idx],\n            'sequence': self.sequence_data[idx]\n        }\n        if self.seq_lengths is not None:\n            sample['seq_length'] = self.seq_lengths[idx]\n        if self.labels is not None:\n            sample['label'] = self.labels[idx]\n        return sample","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.844508Z","iopub.execute_input":"2025-06-28T10:42:52.844775Z","iopub.status.idle":"2025-06-28T10:42:52.863599Z","shell.execute_reply.started":"2025-06-28T10:42:52.844753Z","shell.execute_reply":"2025-06-28T10:42:52.862833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def collate_fn(batch):\n    out = {\n        'static': torch.stack([item['static'] for item in batch]),\n        'sequence': torch.stack([item['sequence'] for item in batch]),\n    }\n\n    if 'seq_length' in batch[0]:\n        out['seq_length'] = torch.tensor([item['seq_length'] for item in batch], dtype=torch.long)\n    if 'label' in batch[0]:\n        out['label'] = torch.stack([item['label'] for item in batch])\n    return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.864732Z","iopub.execute_input":"2025-06-28T10:42:52.865103Z","iopub.status.idle":"2025-06-28T10:42:52.886213Z","shell.execute_reply.started":"2025-06-28T10:42:52.865077Z","shell.execute_reply":"2025-06-28T10:42:52.885604Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.drop(columns=['subject', 'sequence_id'], inplace = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.88683Z","iopub.execute_input":"2025-06-28T10:42:52.887082Z","iopub.status.idle":"2025-06-28T10:42:52.94941Z","shell.execute_reply.started":"2025-06-28T10:42:52.887057Z","shell.execute_reply":"2025-06-28T10:42:52.948827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train.drop(columns=['gesture']), \n                                                    train['gesture'],\n                                                    test_size = 0.2, \n                                                    random_state = 42) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:52.950115Z","iopub.execute_input":"2025-06-28T10:42:52.950351Z","iopub.status.idle":"2025-06-28T10:42:53.085339Z","shell.execute_reply.started":"2025-06-28T10:42:52.95033Z","shell.execute_reply":"2025-06-28T10:42:53.084764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"label_encoder = LabelEncoder()\n\ntrain_labels_string = y_train\ntrain_targets_encoded = label_encoder.fit_transform(train_labels_string)\n\ntest_labels_string = y_test\ntest_targets_encoded = label_encoder.transform(test_labels_string)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:53.086007Z","iopub.execute_input":"2025-06-28T10:42:53.086183Z","iopub.status.idle":"2025-06-28T10:42:53.092326Z","shell.execute_reply.started":"2025-06-28T10:42:53.086169Z","shell.execute_reply":"2025-06-28T10:42:53.09159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Identify dynamic and static columns\ndynamic_cols = [col for col in X_train.columns if isinstance(X_train[col].iloc[0], list)]\nstatic_cols = [col for col in X_train.columns if col not in dynamic_cols]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:53.093103Z","iopub.execute_input":"2025-06-28T10:42:53.093364Z","iopub.status.idle":"2025-06-28T10:42:53.117744Z","shell.execute_reply.started":"2025-06-28T10:42:53.093343Z","shell.execute_reply":"2025-06-28T10:42:53.117093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ----- Preprocess Training Data -----\ntrain_static, train_seq, train_lens, scaler = preprocess_dataframe(\n    X_train, dynamic_cols, static_cols, is_train=True, flatten_dynamic=False  # or False if LSTM\n)\ntrain_labels = torch.tensor(train_targets_encoded, dtype=torch.long)\n\ntrain_dataset = HybridDataset(train_static, train_seq, labels=train_labels, seq_lengths=train_lens)\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, collate_fn=collate_fn)\n\n# ----- Preprocess Test Data -----\ntest_static, test_seq, test_lens, _ = preprocess_dataframe(\n    X_test, dynamic_cols, static_cols, scaler=scaler, is_train=False, flatten_dynamic=False\n)\ntest_labels = torch.tensor(test_targets_encoded, dtype=torch.long)\n\ntest_dataset = HybridDataset(test_static, test_seq, labels=test_labels, seq_lengths=test_lens)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, collate_fn=collate_fn)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:42:53.118594Z","iopub.execute_input":"2025-06-28T10:42:53.119119Z","iopub.status.idle":"2025-06-28T10:44:08.761499Z","shell.execute_reply.started":"2025-06-28T10:42:53.119095Z","shell.execute_reply":"2025-06-28T10:44:08.760538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class HybridCNNLSTM(nn.Module):\n    def __init__(self, static_dim, seq_feat_dim, hidden_dim, num_classes):\n        super(HybridCNNLSTM, self).__init__()\n\n        # Encode static features\n        self.static_encoder = nn.Sequential(\n            nn.Linear(static_dim, hidden_dim),\n            nn.ReLU(),\n            nn.LayerNorm(hidden_dim),\n            nn.Dropout(0.3)  # Stronger regularization\n        )\n\n        # CNN to encode sequential patterns\n        self.cnn = nn.Sequential(\n            nn.Conv1d(in_channels=seq_feat_dim, out_channels=64, kernel_size=5, padding=2),\n            nn.ReLU(),\n            nn.BatchNorm1d(64),\n            nn.Conv1d(64, 128, kernel_size=3, padding=1),\n            nn.ReLU(),\n            nn.BatchNorm1d(128),\n            nn.Dropout(0.4)\n        )\n\n        # BiLSTM for temporal dynamics\n        self.lstm = nn.LSTM(\n            input_size=128,\n            hidden_size=hidden_dim,\n            num_layers=2,\n            batch_first=True,\n            dropout=0.3,\n            bidirectional=True\n        )\n\n        # Optional: LayerNorm after LSTM\n        self.lstm_norm = nn.LayerNorm(hidden_dim * 2)\n\n        # Attention layer\n        self.attention = nn.Sequential(\n            nn.Linear(hidden_dim * 2, hidden_dim),\n            nn.Tanh(),\n            nn.Linear(hidden_dim, 1)\n        )\n\n        # Final classifier\n        self.classifier = nn.Sequential(\n            # nn.Linear(hidden_dim * 2 + hidden_dim, hidden_dim),\n            nn.Linear(hidden_dim * 2, hidden_dim),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(hidden_dim, num_classes)\n        )\n\n    def forward(self, static_input, sequence_input):\n        # Static input: (B, static_dim) -> (B, hidden_dim)\n        static_out = self.static_encoder(static_input)\n\n        # Sequence input: (B, T, F) -> (B, F, T)\n        sequence_input = sequence_input.transpose(1, 2)\n        cnn_out = self.cnn(sequence_input)  # (B, 128, T)\n        cnn_out = cnn_out.transpose(1, 2)   # (B, T, 128)\n\n        # BiLSTM: (B, T, 128) -> (B, T, 2*hidden_dim)\n        lstm_out, _ = self.lstm(cnn_out)\n        lstm_out = self.lstm_norm(lstm_out)\n\n        # Attention mechanism: (B, T, 2*hidden_dim) -> (B, T, 1)\n        attn_scores = self.attention(lstm_out)\n        attn_weights = torch.softmax(attn_scores, dim=1)\n        attn_output = (lstm_out * attn_weights).sum(dim=1)  # (B, 2*hidden_dim)\n\n        # Combine static and temporal outputs\n        # combined = torch.cat((static_out, attn_output), dim=1)\n\n        combined = attn_output\n\n        # Classification\n        return self.classifier(combined)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:44:08.762537Z","iopub.execute_input":"2025-06-28T10:44:08.762836Z","iopub.status.idle":"2025-06-28T10:44:08.772388Z","shell.execute_reply.started":"2025-06-28T10:44:08.76281Z","shell.execute_reply":"2025-06-28T10:44:08.771568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Config\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nnum_classes = len(label_encoder.classes_)\nnum_epochs = 75\n\n# Model instantiation\nmodel = HybridCNNLSTM(\n    static_dim=7,\n    seq_feat_dim=332,\n    hidden_dim=128,\n    num_classes=num_classes\n).to(device)\n\n# Loss + Optimizer\n# criterion = nn.CrossEntropyLoss(label_smoothing=0.1)\n\n# Add weight decay to penalize overconfident weights\noptimizer = optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-4)\n\n# Smarter LR scheduling (optional replacement for ReduceLROnPlateau)\n# scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, factor=0.5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:44:08.773168Z","iopub.execute_input":"2025-06-28T10:44:08.773391Z","iopub.status.idle":"2025-06-28T10:44:09.061372Z","shell.execute_reply.started":"2025-06-28T10:44:08.773375Z","shell.execute_reply":"2025-06-28T10:44:09.060824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.utils.class_weight import compute_class_weight\n\n# Compute weights\nclass_weights = compute_class_weight(\n    class_weight='balanced',\n    classes=np.unique(y_train),   # your integer labels\n    y=y_train\n)\n\n# Convert to tensor\nclass_weights_tensor = torch.tensor(class_weights, dtype=torch.float32).to(device)\n\n# Apply in loss\ncriterion = nn.CrossEntropyLoss(weight=class_weights_tensor, label_smoothing=0.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:44:09.062292Z","iopub.execute_input":"2025-06-28T10:44:09.062546Z","iopub.status.idle":"2025-06-28T10:44:09.073625Z","shell.execute_reply.started":"2025-06-28T10:44:09.062525Z","shell.execute_reply":"2025-06-28T10:44:09.073107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=10, verbose=True, delta=1e-4):\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.delta = delta\n\n    def __call__(self, val_score, model):\n        score = val_score\n\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(model)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            if self.verbose:\n                print(f\"→ No improvement. EarlyStopping counter: {self.counter}/{self.patience}\")\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            if self.verbose:\n                print(f\"✓ Improvement: {score:.4f} (prev best: {self.best_score:.4f})\")\n            self.best_score = score\n            self.save_checkpoint(model)\n            self.counter = 0\n\n    def save_checkpoint(self, model):\n        torch.save(model.state_dict(), \"best_model.pt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:44:09.074325Z","iopub.execute_input":"2025-06-28T10:44:09.074548Z","iopub.status.idle":"2025-06-28T10:44:09.088483Z","shell.execute_reply.started":"2025-06-28T10:44:09.074531Z","shell.execute_reply":"2025-06-28T10:44:09.087717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_acc_history = []\ntest_acc_history = []\ntrain_loss_history = []\ntest_loss_history = []\nbest_test_acc = 0.0\nearly_stopping = EarlyStopping(patience=10, verbose=True)\nfreeze_cnn_epochs = 10\n\nprint(f\"Starting training for {num_epochs} epochs...\")\nprint(f\"Training batches: {len(train_loader)}\")\nprint(f\"Test batches: {len(test_loader)}\")\nprint(\"=\" * 70)\n\nfor epoch in range(num_epochs):\n    model.train()\n    train_losses = []\n    all_preds = []\n    all_labels = []\n\n    if epoch < freeze_cnn_epochs:\n        for param in model.cnn.parameters():\n            param.requires_grad = False\n    else:\n        for param in model.cnn.parameters():\n            param.requires_grad = True\n            \n    for batch_idx, batch in enumerate(train_loader):\n        static_input = batch['static'].to(device)\n        sequence_input = batch['sequence'].to(device)  # shape: (B, D*210)\n        labels = batch['label'].to(device)\n    \n        optimizer.zero_grad()\n        outputs = model(static_input, sequence_input)  # no seq_lengths\n        loss = criterion(outputs, labels)\n        loss.backward()\n        \n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        optimizer.step()\n        \n        train_losses.append(loss.item())\n        preds = torch.argmax(outputs, dim=1)\n        all_preds.extend(preds.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n    \n    train_loss = np.mean(train_losses)\n    train_acc = accuracy_score(all_labels, all_preds)\n    \n    model.eval()\n    test_preds = []\n    test_labels_all = []\n    test_losses = []\n    \n    with torch.no_grad():\n        for batch in test_loader:\n            static_input = batch['static'].to(device)\n            sequence_input = batch['sequence'].to(device)\n            labels = batch['label'].to(device)\n        \n            outputs = model(static_input, sequence_input)\n            test_loss = criterion(outputs, labels)\n            test_losses.append(test_loss.item())\n            \n            preds = torch.argmax(outputs, dim=1)\n            test_preds.extend(preds.cpu().numpy())\n            test_labels_all.extend(labels.cpu().numpy())\n    \n    test_loss = np.mean(test_losses)\n    test_acc = accuracy_score(test_labels_all, test_preds)\n    train_acc_history.append(train_acc)\n    test_acc_history.append(test_acc)\n    train_loss_history.append(train_loss)\n    test_loss_history.append(test_loss)\n    \n    scheduler.step(test_acc)\n    current_lr = optimizer.param_groups[0]['lr']\n    \n    if test_acc > best_test_acc:\n        best_test_acc = test_acc\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'best_test_acc': best_test_acc,\n            'train_acc': train_acc,\n            'test_acc': test_acc,\n            'train_loss': train_loss,\n            'test_loss': test_loss\n        }, 'best_model_checkpoint.pth')\n        print(f\"✓ New best model saved! Test Acc: {test_acc:.4f}\")\n    \n    print(f\"Epoch {epoch+1:2d}/{num_epochs} | \"\n          f\"Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.4f} | \"\n          f\"Test Loss: {test_loss:.4f} | Test Acc: {test_acc:.4f} | \"\n          f\"LR: {current_lr:.2e}\")\n    \n    if current_lr < 1e-6:\n        print(\"Learning rate too small, stopping early...\")\n        break\n\n    early_stopping(test_acc, model)\n    \n    if early_stopping.early_stop:\n        print(\"Early stopping triggered. Training halted.\")\n        break\n\n    print(\"-\" * 70)\n\nprint(\"=\" * 50)\nprint(f\"Training completed!\")\nprint(f\"Best test accuracy: {best_test_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:45:08.725531Z","iopub.execute_input":"2025-06-28T10:45:08.725828Z","iopub.status.idle":"2025-06-28T10:49:45.79287Z","shell.execute_reply.started":"2025-06-28T10:45:08.725808Z","shell.execute_reply":"2025-06-28T10:49:45.792192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:49:45.794309Z","iopub.execute_input":"2025-06-28T10:49:45.794531Z","iopub.status.idle":"2025-06-28T10:49:45.803359Z","shell.execute_reply.started":"2025-06-28T10:49:45.794515Z","shell.execute_reply":"2025-06-28T10:49:45.80276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:49:45.804251Z","iopub.execute_input":"2025-06-28T10:49:45.804871Z","iopub.status.idle":"2025-06-28T10:49:45.822231Z","shell.execute_reply.started":"2025-06-28T10:49:45.804852Z","shell.execute_reply":"2025-06-28T10:49:45.821668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_state_dict(torch.load(\"best_model.pt\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:49:45.823738Z","iopub.execute_input":"2025-06-28T10:49:45.824382Z","iopub.status.idle":"2025-06-28T10:49:45.852882Z","shell.execute_reply.started":"2025-06-28T10:49:45.824359Z","shell.execute_reply":"2025-06-28T10:49:45.852298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    \"\"\"\n    Predict the class label for a given input using a trained model.\n\n    Args:\n        sequence (pl.DataFrame): A Polars DataFrame with dynamic features as columns.\n        demographics (pl.DataFrame): A Polars DataFrame with a single row of static features.\n\n    Returns:\n        str: Predicted class label.\n    \"\"\"\n    # 1. Extract and stack dynamic features\n    seq_np = np.column_stack([sequence[col].to_numpy() for col in dynamic_cols])\n    seq_np = np.where(seq_np == -1, 0.0, seq_np)        # replace -1 with 0\n    seq_np = np.nan_to_num(seq_np, nan=0.0)             # replace NaN with 0\n\n    # 2. Extract static features\n    static_np = demographics.select(static_cols).to_numpy()[0]\n\n    # 3. Truncate or pad sequence to fixed length\n    if len(seq_np) > MAX_SEQ_LEN:\n        seq_np = seq_np[:MAX_SEQ_LEN]\n    elif len(seq_np) < MAX_SEQ_LEN:\n        pad = np.zeros((MAX_SEQ_LEN - len(seq_np), seq_np.shape[1]), dtype=np.float32)\n        seq_np = np.vstack([seq_np, pad])\n\n    # 4. Convert to torch tensors\n    seq_tensor = torch.tensor(seq_np, dtype=torch.float32).unsqueeze(0).to(device)        # (1, MAX_SEQ_LEN, seq_feat_dim)\n    static_tensor = torch.tensor(static_np, dtype=torch.float32).unsqueeze(0).to(device)  # (1, static_feat_dim)\n\n    # 5. Predict\n    model.eval()\n    with torch.no_grad():\n        logits = model(static_tensor, seq_tensor)\n        pred_class = torch.argmax(logits, dim=1).item()\n\n    # 6. Decode label\n    return label_encoder.inverse_transform([pred_class])[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:49:45.853578Z","iopub.execute_input":"2025-06-28T10:49:45.853821Z","iopub.status.idle":"2025-06-28T10:49:45.860149Z","shell.execute_reply.started":"2025-06-28T10:49:45.853799Z","shell.execute_reply":"2025-06-28T10:49:45.859419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv('KAGGLE_IS_COMPETITION_RERUN'):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv',\n            '/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv',\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:49:45.861048Z","iopub.execute_input":"2025-06-28T10:49:45.861491Z","iopub.status.idle":"2025-06-28T10:49:47.485385Z","shell.execute_reply.started":"2025-06-28T10:49:45.861469Z","shell.execute_reply":"2025-06-28T10:49:47.484758Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    # Replace this function with your inference code.\n    # You can return either a Pandas or Polars dataframe, though Polars is recommended.\n    # Each prediction (except the very first) must be returned within 30 minutes of the batch features being provided.\n #   return 'Text on phone'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:44:09.151065Z","iopub.status.idle":"2025-06-28T10:44:09.151273Z","shell.execute_reply.started":"2025-06-28T10:44:09.15118Z","shell.execute_reply":"2025-06-28T10:44:09.151189Z"}},"outputs":[],"execution_count":null}]}