{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31090,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:17:33.505499Z","iopub.execute_input":"2025-08-09T08:17:33.506311Z","iopub.status.idle":"2025-08-09T08:17:33.775518Z","shell.execute_reply.started":"2025-08-09T08:17:33.506283Z","shell.execute_reply":"2025-08-09T08:17:33.774935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1.1: Import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set random seeds for reproducibility\nnp.random.seed(42)\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n torch.cuda.manual_seed(42)\n\nprint(\"Libraries imported successfully!\")\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:17:33.77673Z","iopub.execute_input":"2025-08-09T08:17:33.777037Z","iopub.status.idle":"2025-08-09T08:17:38.626785Z","shell.execute_reply.started":"2025-08-09T08:17:33.777019Z","shell.execute_reply":"2025-08-09T08:17:38.626159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1.2: Load the dataset\ntrain = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demo = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\ntest = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\n\nprint(\"Dataset loaded successfully!\")\nprint(f\"Train shape: {train.shape}\")\nprint(f\"Train demographics shape: {train_demo.shape}\")\nprint(f\"Test shape: {test.shape}\")\nprint(f\"\\nUnique subjects: {train['subject'].nunique()}\")\nprint(f\"Unique sequences: {train['sequence_id'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:17:38.627403Z","iopub.execute_input":"2025-08-09T08:17:38.627708Z","iopub.status.idle":"2025-08-09T08:18:14.267387Z","shell.execute_reply.started":"2025-08-09T08:17:38.627691Z","shell.execute_reply":"2025-08-09T08:18:14.266599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2.1: Analyze target distribution\nplt.figure(figsize=(15, 6))\ngesture_counts = train['gesture'].value_counts()\nplt.subplot(1, 2, 1)\ngesture_counts.plot(kind='bar', rot=45)\nplt.title('Distribution of Gestures')\nplt.xlabel('Gesture')\nplt.ylabel('Count')\n\n# Check sequence lengths\nplt.subplot(1, 2, 2)\nseq_lengths = train.groupby('sequence_id').size()\nplt.hist(seq_lengths, bins=30, alpha=0.7)\nplt.title('Distribution of Sequence Lengths')\nplt.xlabel('Sequence Length')\nplt.ylabel('Frequency')\nplt.tight_layout()\nplt.show()\n\nprint(f\"Sequence length statistics:\")\nprint(f\"Mean: {seq_lengths.mean():.2f}\")\nprint(f\"Median: {seq_lengths.median():.2f}\")\nprint(f\"Min: {seq_lengths.min()}\")\nprint(f\"Max: {seq_lengths.max()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:14.268119Z","iopub.execute_input":"2025-08-09T08:18:14.268337Z","iopub.status.idle":"2025-08-09T08:18:14.911719Z","shell.execute_reply.started":"2025-08-09T08:18:14.268318Z","shell.execute_reply":"2025-08-09T08:18:14.910953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2.2: Analyze sensor features\n# Define feature groups\nimu_cols = [col for col in train.columns if col.startswith('acc_') or col.startswith('rot_')]\nthm_cols = [col for col in train.columns if col.startswith('thm_')]\ntof_cols = [col for col in train.columns if col.startswith('tof_')]\n\nprint(f\"Feature groups:\")\nprint(f\"IMU features: {len(imu_cols)}\")\nprint(f\"Thermopile features: {len(thm_cols)}\")\nprint(f\"ToF features: {len(tof_cols)}\")\nprint(f\"Total sensor features: {len(imu_cols + thm_cols + tof_cols)}\")\n\n# Check for missing values\nmissing_values = train[imu_cols + thm_cols + tof_cols].isnull().sum().sum()\nnegative_ones = (train[imu_cols + thm_cols + tof_cols] == -1).sum().sum()\nprint(f\"\\nMissing values (NaN): {missing_values}\")\nprint(f\"Missing values (-1): {negative_ones}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:14.913149Z","iopub.execute_input":"2025-08-09T08:18:14.913391Z","iopub.status.idle":"2025-08-09T08:18:16.598175Z","shell.execute_reply.started":"2025-08-09T08:18:14.913351Z","shell.execute_reply":"2025-08-09T08:18:16.59743Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3.1: Extract sequences and labels\nfeature_cols = imu_cols + thm_cols + tof_cols\ntarget_col = 'gesture'\n\n# Group by sequence_id and extract sequences\ngrouped = train.groupby('sequence_id')\n\nX_sequences = []\ny_sequences = []\nsubject_sequences = []\n\nprint(\"Extracting sequences...\")\nfor seq_id, group in tqdm(grouped):\n features = group[feature_cols].values # shape: (seq_len, feature_dim)\n label = group[target_col].iloc[0] # gesture is the same for all rows\n subject = group['subject'].iloc[0] # subject is the same for all rows\n \n X_sequences.append(features)\n y_sequences.append(label)\n subject_sequences.append(subject)\n\nprint(f\"Extracted {len(X_sequences)} sequences\")\nprint(f\"Example sequence shape: {X_sequences[0].shape}\")\nprint(f\"First few labels: {y_sequences[:5]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:16.59902Z","iopub.execute_input":"2025-08-09T08:18:16.599339Z","iopub.status.idle":"2025-08-09T08:18:22.14931Z","shell.execute_reply.started":"2025-08-09T08:18:16.599319Z","shell.execute_reply":"2025-08-09T08:18:22.148571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3.2: Handle missing values and sequence padding\nSEQ_LEN = 60 # Fixed sequence length\n\ndef pad_sequence(seq, max_len=SEQ_LEN):\n \"\"\"Pad or truncate sequence to fixed length\"\"\"\n # Handle missing values (-1 and NaN)\n seq = np.where(np.isnan(seq), 0, seq)\n seq = np.where(seq == -1, 0, seq)\n \n if len(seq) >= max_len:\n return seq[:max_len] # Truncate\n else:\n # Pad with zeros\n pad = np.zeros((max_len - len(seq), seq.shape[1]))\n return np.vstack([seq, pad])\n\nprint(\"Padding sequences...\")\nX_padded = []\nfor seq in tqdm(X_sequences):\n X_padded.append(pad_sequence(seq, SEQ_LEN))\n\nX_padded = np.stack(X_padded)\nprint(f\"Padded sequences shape: {X_padded.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:22.149968Z","iopub.execute_input":"2025-08-09T08:18:22.150145Z","iopub.status.idle":"2025-08-09T08:18:25.167537Z","shell.execute_reply.started":"2025-08-09T08:18:22.15013Z","shell.execute_reply":"2025-08-09T08:18:25.166864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 3.3: Feature normalization\nprint(\"Normalizing features...\")\nn_seq, seq_len, n_feat = X_padded.shape\n\n# Reshape for normalization\nX_padded_2d = X_padded.reshape(-1, n_feat)\n\n# Fit scaler and transform\nscaler = StandardScaler()\nX_padded_2d_scaled = scaler.fit_transform(X_padded_2d)\n\n# Reshape back to 3D\nX_padded_norm = X_padded_2d_scaled.reshape(n_seq, seq_len, n_feat)\nprint(f\"Normalized sequences shape: {X_padded_norm.shape}\")\n\n# Check normalization\nprint(f\"Feature means after normalization: {X_padded_2d_scaled.mean(axis=0)[:5]}\")\nprint(f\"Feature stds after normalization: {X_padded_2d_scaled.std(axis=0)[:5]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:25.168223Z","iopub.execute_input":"2025-08-09T08:18:25.168496Z","iopub.status.idle":"2025-08-09T08:18:28.432895Z","shell.execute_reply.started":"2025-08-09T08:18:25.168468Z","shell.execute_reply":"2025-08-09T08:18:28.43221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4.1: Process demographic features\ndemo_feature_cols = [\n 'adult_child', 'age', 'sex', 'handedness',\n 'height_cm', 'shoulder_to_wrist_cm', 'elbow_to_wrist_cm'\n]\n\nprint(\"Demographic features:\", demo_feature_cols)\nprint(\"\\nDemographic data info:\")\nprint(train_demo[demo_feature_cols].describe())\n\n# Create demographic mapping\ndemo_map = train_demo.set_index('subject')[demo_feature_cols].to_dict(orient='index')\n\n# Extract demographic features for each sequence\ndemo_arrays = []\nfor subject in subject_sequences:\n demo_data = demo_map.get(subject, {k: 0 for k in demo_feature_cols})\n demo_array = np.array(list(demo_data.values()), dtype=float)\n demo_arrays.append(demo_array)\n\ndemo_arrays = np.stack(demo_arrays)\nprint(f\"Demographic arrays shape: {demo_arrays.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:28.433601Z","iopub.execute_input":"2025-08-09T08:18:28.43407Z","iopub.status.idle":"2025-08-09T08:18:28.48664Z","shell.execute_reply.started":"2025-08-09T08:18:28.434049Z","shell.execute_reply":"2025-08-09T08:18:28.485914Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4.2: Encode categorical demographic features\nfrom sklearn.preprocessing import LabelEncoder\n\n# Encode categorical features\ndemo_df = pd.DataFrame(demo_arrays, columns=demo_feature_cols)\n\n# Encode categorical columns\ncategorical_cols = ['adult_child', 'sex', 'handedness']\nfor col in categorical_cols:\n if col in demo_df.columns:\n le = LabelEncoder()\n demo_df[col] = le.fit_transform(demo_df[col].astype(str))\n\n# Scale demographic features\ndemo_scaler = StandardScaler()\ndemo_scaled = demo_scaler.fit_transform(demo_df.values)\n\nprint(f\"Scaled demographic features shape: {demo_scaled.shape}\")\nprint(f\"Demographic features mean: {demo_scaled.mean(axis=0)}\")\nprint(f\"Demographic features std: {demo_scaled.std(axis=0)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:28.487301Z","iopub.execute_input":"2025-08-09T08:18:28.487505Z","iopub.status.idle":"2025-08-09T08:18:28.510348Z","shell.execute_reply.started":"2025-08-09T08:18:28.487488Z","shell.execute_reply":"2025-08-09T08:18:28.509782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 4.3: Augment sequences with demographic features\nprint(\"Augmenting sequences with demographic features...\")\nX_augmented = []\n\nfor seq, demo in tqdm(zip(X_padded_norm, demo_scaled), total=len(X_padded_norm)):\n # Repeat demographic features for each time step\n demo_seq = np.tile(demo, (seq.shape[0], 1)) # (SEQ_LEN, n_demo_feat)\n # Concatenate sensor features with demographic features\n seq_plus_demo = np.concatenate([seq, demo_seq], axis=1)\n X_augmented.append(seq_plus_demo)\n\nX_augmented = np.stack(X_augmented)\nprint(f\"Augmented sequences shape: {X_augmented.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:28.511099Z","iopub.execute_input":"2025-08-09T08:18:28.511469Z","iopub.status.idle":"2025-08-09T08:18:29.230198Z","shell.execute_reply.started":"2025-08-09T08:18:28.51145Z","shell.execute_reply":"2025-08-09T08:18:29.229413Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 5.1: Encode target labels\nle_target = LabelEncoder()\ny_encoded = le_target.fit_transform(y_sequences)\n\nprint(f\"Target classes: {le_target.classes_}\")\nprint(f\"Number of classes: {len(le_target.classes_)}\")\nprint(f\"Encoded labels shape: {y_encoded.shape}\")\nprint(f\"Label distribution: {np.bincount(y_encoded)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:29.231056Z","iopub.execute_input":"2025-08-09T08:18:29.231333Z","iopub.status.idle":"2025-08-09T08:18:29.241881Z","shell.execute_reply.started":"2025-08-09T08:18:29.231308Z","shell.execute_reply":"2025-08-09T08:18:29.241186Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 5.2: Create train-validation split\n# Use stratified split to maintain class distribution\nX_train, X_val, y_train, y_val = train_test_split(\n X_augmented, y_encoded, \n test_size=0.2, \n random_state=42, \n stratify=y_encoded\n)\n\nprint(f\"Training set shape: {X_train.shape}\")\nprint(f\"Validation set shape: {X_val.shape}\")\nprint(f\"Training labels shape: {y_train.shape}\")\nprint(f\"Validation labels shape: {y_val.shape}\")\n\n# Check class distribution in splits\nprint(f\"\\nTraining class distribution: {np.bincount(y_train)}\")\nprint(f\"Validation class distribution: {np.bincount(y_val)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:29.242724Z","iopub.execute_input":"2025-08-09T08:18:29.243409Z","iopub.status.idle":"2025-08-09T08:18:29.612137Z","shell.execute_reply.started":"2025-08-09T08:18:29.243364Z","shell.execute_reply":"2025-08-09T08:18:29.61155Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 5.3: Create PyTorch datasets and dataloaders\nclass BehaviorDataset(Dataset):\n def __init__(self, sequences, labels):\n self.sequences = torch.FloatTensor(sequences)\n self.labels = torch.LongTensor(labels)\n \n def __len__(self):\n return len(self.sequences)\n \n def __getitem__(self, idx):\n return self.sequences[idx], self.labels[idx]\n\n# Create datasets\ntrain_dataset = BehaviorDataset(X_train, y_train)\nval_dataset = BehaviorDataset(X_val, y_val)\n\n# Create dataloaders\nBATCH_SIZE = 32\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n\nprint(f\"Training batches: {len(train_loader)}\")\nprint(f\"Validation batches: {len(val_loader)}\")\n\n# Test the dataloader\nsample_batch = next(iter(train_loader))\nprint(f\"Sample batch shape: {sample_batch[0].shape}, {sample_batch[1].shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:29.614983Z","iopub.execute_input":"2025-08-09T08:18:29.615199Z","iopub.status.idle":"2025-08-09T08:18:29.939057Z","shell.execute_reply.started":"2025-08-09T08:18:29.615182Z","shell.execute_reply":"2025-08-09T08:18:29.938335Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.4: Fixed Multi-Scale Convolution Module\nclass MultiScaleConv1D(nn.Module):\n def __init__(self, in_channels, out_channels, kernel_sizes=[3, 7, 15]):\n super().__init__()\n self.kernel_sizes = kernel_sizes\n self.out_channels = out_channels\n \n # Calculate output channels per kernel to sum to exact out_channels\n base_channels = out_channels // len(kernel_sizes)\n remainder = out_channels % len(kernel_sizes)\n \n self.out_channels_per_kernel = [base_channels] * len(kernel_sizes)\n # Distribute remainder channels to first few kernels\n for i in range(remainder):\n self.out_channels_per_kernel[i] += 1\n \n self.convs = nn.ModuleList([\n nn.Conv1d(in_channels, out_ch, \n kernel_size=k, padding=k//2, bias=False)\n for k, out_ch in zip(kernel_sizes, self.out_channels_per_kernel)\n ])\n \n self.batch_norms = nn.ModuleList([\n nn.BatchNorm1d(out_ch) for out_ch in self.out_channels_per_kernel\n ])\n \n self.activation = nn.ReLU()\n \n # Add a final projection to ensure exact output dimension\n total_conv_output = sum(self.out_channels_per_kernel)\n if total_conv_output != out_channels:\n self.final_projection = nn.Linear(total_conv_output, out_channels)\n else:\n self.final_projection = None\n \n def forward(self, x):\n # x: (batch, seq_len, features) -> (batch, features, seq_len)\n x = x.transpose(1, 2)\n \n outputs = []\n for conv, bn in zip(self.convs, self.batch_norms):\n conv_out = conv(x)\n bn_out = bn(conv_out)\n activated = self.activation(bn_out)\n outputs.append(activated)\n \n # Concatenate along channel dimension\n combined = torch.cat(outputs, dim=1)\n # Transpose back: (batch, features, seq_len) -> (batch, seq_len, features)\n combined = combined.transpose(1, 2)\n \n # Apply final projection if needed\n if self.final_projection is not None:\n combined = self.final_projection(combined)\n \n return combined\n\n# Test the fixed module\ntest_input = torch.randn(4, 60, 339)\nmulti_scale_fixed = MultiScaleConv1D(339, 256)\ntest_output = multi_scale_fixed(test_input)\nprint(f\"Fixed multi-scale output shape: {test_output.shape}\")\nprint(f\"Expected shape: (4, 60, 256)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:29.940091Z","iopub.execute_input":"2025-08-09T08:18:29.940325Z","iopub.status.idle":"2025-08-09T08:18:30.126453Z","shell.execute_reply.started":"2025-08-09T08:18:29.9403Z","shell.execute_reply":"2025-08-09T08:18:30.125732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.2: Hierarchical Attention Module\nclass HierarchicalAttention(nn.Module):\n def __init__(self, d_model, n_heads=8, dropout=0.1):\n super().__init__()\n \n # Feature-level attention (across feature dimension)\n self.feature_attention = nn.MultiheadAttention(\n d_model, n_heads, dropout=dropout, batch_first=True\n )\n \n # Temporal attention (across time dimension)\n self.temporal_attention = nn.MultiheadAttention(\n d_model, n_heads, dropout=dropout, batch_first=True\n )\n \n # Layer normalization\n self.layer_norm1 = nn.LayerNorm(d_model)\n self.layer_norm2 = nn.LayerNorm(d_model)\n \n # Feed-forward network\n self.ffn = nn.Sequential(\n nn.Linear(d_model, d_model * 4),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(d_model * 4, d_model)\n )\n \n self.dropout = nn.Dropout(dropout)\n \n def forward(self, x):\n batch_size, seq_len, d_model = x.shape\n \n # Temporal attention\n attn_output, _ = self.temporal_attention(x, x, x)\n x = self.layer_norm1(x + self.dropout(attn_output))\n \n # Feed-forward network\n ffn_output = self.ffn(x)\n x = self.layer_norm2(x + self.dropout(ffn_output))\n \n return x\n\n# Test the hierarchical attention\ntest_input = torch.randn(4, 60, 256)\nhier_attention = HierarchicalAttention(256, n_heads=8)\ntest_output = hier_attention(test_input)\nprint(f\"Hierarchical attention output shape: {test_output.shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:30.127164Z","iopub.execute_input":"2025-08-09T08:18:30.127394Z","iopub.status.idle":"2025-08-09T08:18:30.201266Z","shell.execute_reply.started":"2025-08-09T08:18:30.127354Z","shell.execute_reply":"2025-08-09T08:18:30.20068Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 6.5: Updated Complete HMTAN Model\nclass HMTAN(nn.Module):\n def __init__(self, input_dim=339, d_model=256, n_heads=8, n_layers=4, n_classes=18, dropout=0.1):\n super().__init__()\n \n self.input_dim = input_dim\n self.d_model = d_model\n \n # Multi-scale feature extraction (fixed)\n self.multi_scale_conv = MultiScaleConv1D(input_dim, d_model)\n \n # Positional encoding\n self.register_buffer('pos_encoding', self._create_positional_encoding(60, d_model))\n \n # Transformer encoder layers with hierarchical attention\n self.transformer_layers = nn.ModuleList([\n HierarchicalAttention(d_model, n_heads, dropout)\n for _ in range(n_layers)\n ])\n \n # Global attention pooling\n self.global_attention = nn.Sequential(\n nn.Linear(d_model, 1),\n nn.Softmax(dim=1)\n )\n \n # Classification head\n self.classifier = nn.Sequential(\n nn.Linear(d_model, d_model // 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(d_model // 2, n_classes)\n )\n \n def _create_positional_encoding(self, max_len, d_model):\n pe = torch.zeros(max_len, d_model)\n position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n \n div_term = torch.exp(torch.arange(0, d_model, 2).float() * \n (-np.log(10000.0) / d_model))\n \n pe[:, 0::2] = torch.sin(position * div_term)\n if d_model % 2 == 1: # Handle odd d_model\n pe[:, 1::2] = torch.cos(position * div_term[:-1])\n else:\n pe[:, 1::2] = torch.cos(position * div_term)\n \n return pe.unsqueeze(0) # Shape: (1, max_len, d_model)\n \n def forward(self, x):\n batch_size, seq_len, _ = x.shape\n \n # Multi-scale feature extraction\n x = self.multi_scale_conv(x) # (batch, seq_len, d_model)\n \n # Add positional encoding\n x = x + self.pos_encoding[:, :seq_len, :]\n \n # Apply transformer layers with hierarchical attention\n for layer in self.transformer_layers:\n x = layer(x)\n \n # Global attention pooling\n attention_weights = self.global_attention(x) # (batch, seq_len, 1)\n x = torch.sum(x * attention_weights, dim=1) # (batch, d_model)\n \n # Classification\n return self.classifier(x)\n\n# Initialize the fixed model\nmodel = HMTAN(input_dim=X_augmented.shape[2], n_classes=len(le_target.classes_))\nprint(f\"Model initialized with {sum(p.numel() for p in model.parameters())} parameters\")\n\n# Test forward pass with fixed model\ntest_batch = next(iter(train_loader))\nprint(f\"Test batch input shape: {test_batch[0].shape}\")\nwith torch.no_grad():\n output = model(test_batch[0])\n print(f\"Model output shape: {output.shape}\")\n print(f\"Sample predictions: {torch.softmax(output, dim=1)[0][:5]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:30.202121Z","iopub.execute_input":"2025-08-09T08:18:30.202363Z","iopub.status.idle":"2025-08-09T08:18:30.586042Z","shell.execute_reply.started":"2025-08-09T08:18:30.202345Z","shell.execute_reply":"2025-08-09T08:18:30.585263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 7.1: Training utilities and metrics\nimport time\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n\ndef calculate_accuracy(outputs, labels):\n _, predicted = torch.max(outputs.data, 1)\n total = labels.size(0)\n correct = (predicted == labels).sum().item()\n return correct / total\n\nclass EarlyStopping:\n def __init__(self, patience=7, min_delta=0.001):\n self.patience = patience\n self.min_delta = min_delta\n self.counter = 0\n self.best_loss = float('inf')\n \n def __call__(self, val_loss):\n if val_loss < self.best_loss - self.min_delta:\n self.best_loss = val_loss\n self.counter = 0\n return False\n else:\n self.counter += 1\n return self.counter >= self.patience\n\ndef train_epoch(model, train_loader, criterion, optimizer, device):\n model.train()\n total_loss = 0\n total_correct = 0\n total_samples = 0\n \n for batch_x, batch_y in tqdm(train_loader, desc=\"Training\"):\n batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n \n optimizer.zero_grad()\n outputs = model(batch_x)\n loss = criterion(outputs, batch_y)\n loss.backward()\n \n # Gradient clipping\n torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n \n optimizer.step()\n \n total_loss += loss.item()\n total_correct += calculate_accuracy(outputs, batch_y) * batch_y.size(0)\n total_samples += batch_y.size(0)\n \n return total_loss / len(train_loader), total_correct / total_samples\n\ndef validate_epoch(model, val_loader, criterion, device):\n model.eval()\n total_loss = 0\n all_predictions = []\n all_labels = []\n \n with torch.no_grad():\n for batch_x, batch_y in tqdm(val_loader, desc=\"Validation\"):\n batch_x, batch_y = batch_x.to(device), batch_y.to(device)\n \n outputs = model(batch_x)\n loss = criterion(outputs, batch_y)\n \n total_loss += loss.item()\n \n _, predicted = torch.max(outputs.data, 1)\n all_predictions.extend(predicted.cpu().numpy())\n all_labels.extend(batch_y.cpu().numpy())\n \n accuracy = accuracy_score(all_labels, all_predictions)\n return total_loss / len(val_loader), accuracy, all_predictions, all_labels\n\nprint(\"Training utilities defined successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:30.587081Z","iopub.execute_input":"2025-08-09T08:18:30.587287Z","iopub.status.idle":"2025-08-09T08:18:30.597596Z","shell.execute_reply.started":"2025-08-09T08:18:30.587265Z","shell.execute_reply":"2025-08-09T08:18:30.596789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 7.2: Initialize training components\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Move model to device\nmodel = model.to(device)\n\n# Loss function with class weights for imbalanced data\nclass_counts = np.bincount(y_train)\nclass_weights = len(y_train) / (len(class_counts) * class_counts)\nclass_weights = torch.FloatTensor(class_weights).to(device)\n\ncriterion = nn.CrossEntropyLoss(weight=class_weights)\noptimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=1e-5)\nscheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=50)\n\n# Early stopping\nearly_stopping = EarlyStopping(patience=10, min_delta=0.001)\n\nprint(\"Training components initialized!\")\nprint(f\"Class weights: {class_weights[:5]}...\") # Show first 5 weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:30.598224Z","iopub.execute_input":"2025-08-09T08:18:30.5985Z","iopub.status.idle":"2025-08-09T08:18:33.593187Z","shell.execute_reply.started":"2025-08-09T08:18:30.598483Z","shell.execute_reply":"2025-08-09T08:18:33.592418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 7.3: Training loop\nn_epochs = 50\ntrain_losses = []\ntrain_accuracies = []\nval_losses = []\nval_accuracies = []\n\nbest_val_acc = 0\nbest_model_state = None\n\nprint(\"Starting training...\")\nstart_time = time.time()\n\nfor epoch in range(n_epochs):\n epoch_start = time.time()\n \n # Train\n train_loss, train_acc = train_epoch(model, train_loader, criterion, optimizer, device)\n \n # Validate\n val_loss, val_acc, val_preds, val_labels = validate_epoch(model, val_loader, criterion, device)\n \n # Update learning rate\n scheduler.step()\n \n # Store metrics\n train_losses.append(train_loss)\n train_accuracies.append(train_acc)\n val_losses.append(val_loss)\n val_accuracies.append(val_acc)\n \n # Save best model\n if val_acc > best_val_acc:\n best_val_acc = val_acc\n best_model_state = model.state_dict().copy()\n torch.save(best_model_state, 'hmtan_best.pth')\n \n epoch_time = time.time() - epoch_start\n \n print(f'Epoch [{epoch+1}/{n_epochs}] ({epoch_time:.1f}s)')\n print(f' Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}')\n print(f' Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.4f}')\n print(f' LR: {optimizer.param_groups[0][\"lr\"]:.6f}')\n print('-' * 60)\n \n # Early stopping\n if early_stopping(val_loss):\n print(f'Early stopping triggered at epoch {epoch+1}')\n break\n\ntotal_time = time.time() - start_time\nprint(f'Training completed in {total_time/60:.1f} minutes')\nprint(f'Best validation accuracy: {best_val_acc:.4f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:18:33.594111Z","iopub.execute_input":"2025-08-09T08:18:33.594707Z","iopub.status.idle":"2025-08-09T08:20:22.056938Z","shell.execute_reply.started":"2025-08-09T08:18:33.594674Z","shell.execute_reply":"2025-08-09T08:20:22.056136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 8.1: Fixed MultiScaleConvWithAttention\nclass MultiScaleConvWithAttention(nn.Module):\n def __init__(self, d_model, out_dim, dropout=0.1):\n super().__init__()\n \n # Multiple kernel sizes for different temporal patterns\n self.kernel_sizes = [3, 5, 7, 9] # Reduced to 4 for cleaner division\n \n # Calculate exact output channels per kernel\n channels_per_kernel = out_dim // len(self.kernel_sizes)\n remaining_channels = out_dim % len(self.kernel_sizes)\n \n self.out_channels_per_kernel = [channels_per_kernel] * len(self.kernel_sizes)\n # Add remaining channels to first kernels\n for i in range(remaining_channels):\n self.out_channels_per_kernel[i] += 1\n \n self.convs = nn.ModuleList([\n nn.Conv1d(d_model, out_ch, kernel_size=k, padding=k//2)\n for k, out_ch in zip(self.kernel_sizes, self.out_channels_per_kernel)\n ])\n \n # Total output channels for attention\n total_out_channels = sum(self.out_channels_per_kernel)\n \n # Attention to weight different scales\n self.scale_attention = nn.Sequential(\n nn.Linear(total_out_channels, total_out_channels),\n nn.ReLU(),\n nn.Linear(total_out_channels, len(self.kernel_sizes)),\n nn.Softmax(dim=-1)\n )\n \n # Final projection to ensure exact output dimension\n self.final_projection = nn.Linear(total_out_channels, out_dim)\n self.dropout = nn.Dropout(dropout)\n \n def forward(self, x):\n # x: (batch, seq_len, d_model)\n x_transposed = x.transpose(1, 2) # (batch, d_model, seq_len)\n \n # Apply multi-scale convolutions\n scale_outputs = []\n for conv in self.convs:\n scale_out = F.relu(conv(x_transposed)) # Add activation\n scale_outputs.append(scale_out)\n \n # Concatenate scales\n combined = torch.cat(scale_outputs, dim=1) # (batch, total_out_channels, seq_len)\n combined = combined.transpose(1, 2) # (batch, seq_len, total_out_channels)\n \n # Apply final projection to get exact output dimension\n result = self.final_projection(combined) # (batch, seq_len, out_dim)\n \n return self.dropout(result)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:55:16.365043Z","iopub.execute_input":"2025-08-09T08:55:16.365539Z","iopub.status.idle":"2025-08-09T08:55:16.374147Z","shell.execute_reply.started":"2025-08-09T08:55:16.365509Z","shell.execute_reply":"2025-08-09T08:55:16.373294Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 8.2: Simplified but robust model architecture\nclass RobustHMTAN(nn.Module):\n def __init__(self, input_dim=339, d_model=256, n_heads=8, n_layers=4, \n n_classes=18, dropout=0.15):\n super().__init__()\n \n self.input_dim = input_dim\n self.d_model = d_model\n \n # Input projection with proper dimensions\n self.input_projection = nn.Linear(input_dim, d_model)\n self.input_norm = nn.LayerNorm(d_model)\n self.input_dropout = nn.Dropout(dropout)\n \n # Multi-scale temporal convolutions (simpler approach)\n self.temporal_conv1 = nn.Conv1d(d_model, d_model, kernel_size=3, padding=1)\n self.temporal_conv2 = nn.Conv1d(d_model, d_model, kernel_size=5, padding=2)\n self.temporal_conv3 = nn.Conv1d(d_model, d_model, kernel_size=7, padding=3)\n \n # Combine multi-scale features\n self.feature_fusion = nn.Linear(d_model * 3, d_model)\n self.fusion_norm = nn.LayerNorm(d_model)\n \n # Positional encoding\n self.register_buffer('pos_encoding', self._create_positional_encoding(60, d_model))\n \n # Transformer encoder layers\n encoder_layer = nn.TransformerEncoderLayer(\n d_model=d_model,\n nhead=n_heads,\n dim_feedforward=d_model * 4,\n dropout=dropout,\n activation='gelu',\n batch_first=True,\n norm_first=True\n )\n self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=n_layers)\n \n # Global pooling\n self.global_pool = nn.AdaptiveAvgPool1d(1)\n \n # Classification head\n self.classifier = nn.Sequential(\n nn.Linear(d_model, d_model // 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(d_model // 2, d_model // 4),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(d_model // 4, n_classes)\n )\n \n def _create_positional_encoding(self, max_len, d_model):\n pe = torch.zeros(max_len, d_model)\n position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n \n div_term = torch.exp(torch.arange(0, d_model, 2).float() * \n (-np.log(10000.0) / d_model))\n \n pe[:, 0::2] = torch.sin(position * div_term)\n if d_model % 2 == 1:\n pe[:, 1::2] = torch.cos(position * div_term[:-1])\n else:\n pe[:, 1::2] = torch.cos(position * div_term)\n \n return pe.unsqueeze(0)\n \n def forward(self, x):\n batch_size, seq_len, _ = x.shape\n \n # Input projection and normalization\n x = self.input_projection(x) # (batch, seq_len, d_model)\n x = self.input_norm(x)\n x = self.input_dropout(x)\n \n # Multi-scale temporal convolutions\n x_transposed = x.transpose(1, 2) # (batch, d_model, seq_len)\n \n conv1_out = F.relu(self.temporal_conv1(x_transposed))\n conv2_out = F.relu(self.temporal_conv2(x_transposed))\n conv3_out = F.relu(self.temporal_conv3(x_transposed))\n \n # Concatenate multi-scale features\n multi_scale = torch.cat([conv1_out, conv2_out, conv3_out], dim=1)\n multi_scale = multi_scale.transpose(1, 2) # (batch, seq_len, d_model*3)\n \n # Fuse multi-scale features\n x = self.feature_fusion(multi_scale) # (batch, seq_len, d_model)\n x = self.fusion_norm(x)\n \n # Add positional encoding\n x = x + self.pos_encoding[:, :seq_len, :]\n \n # Apply transformer\n x = self.transformer(x)\n \n # Global pooling\n x = x.transpose(1, 2) # (batch, d_model, seq_len)\n x = self.global_pool(x).squeeze(-1) # (batch, d_model)\n \n # Classification\n return self.classifier(x)\n\n# Test the robust model\nprint(\"Testing RobustHMTAN model...\")\ntest_input = torch.randn(4, 60, 339)\nrobust_model = RobustHMTAN(input_dim=339, n_classes=18)\nwith torch.no_grad():\n test_output = robust_model(test_input)\n print(f\"Test output shape: {test_output.shape}\")\n print(\"Model test passed!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T08:55:46.476217Z","iopub.execute_input":"2025-08-09T08:55:46.476877Z","iopub.status.idle":"2025-08-09T08:55:46.566544Z","shell.execute_reply.started":"2025-08-09T08:55:46.47685Z","shell.execute_reply":"2025-08-09T08:55:46.565835Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 8.3: Robust training function with error handling\ndef train_robust_model():\n device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n print(f\"Using device: {device}\")\n \n # Create datasets with augmentation\n train_dataset_aug = AdvancedBehaviorDataset(\n X_train, y_train, augment=True, augment_prob=0.6\n )\n val_dataset = AdvancedBehaviorDataset(\n X_val, y_val, augment=False\n )\n \n # Create data loaders\n train_loader = DataLoader(\n train_dataset_aug, \n batch_size=24, # Moderate batch size\n shuffle=True, \n num_workers=2,\n pin_memory=True if device.type == 'cuda' else False\n )\n \n val_loader = DataLoader(\n val_dataset, \n batch_size=32, \n shuffle=False, \n num_workers=2,\n pin_memory=True if device.type == 'cuda' else False\n )\n \n # Create robust model\n model = RobustHMTAN(\n input_dim=X_augmented.shape[2], \n d_model=256, # Smaller model for stability\n n_heads=8,\n n_layers=4,\n n_classes=len(le_target.classes_),\n dropout=0.15\n ).to(device)\n \n print(f\"Model parameters: {sum(p.numel() for p in model.parameters()):,}\")\n \n # Loss function with class weights\n class_weights = torch.FloatTensor([\n len(y_train) / (len(np.unique(y_train)) * max(1, np.bincount(y_train)[i])) \n for i in range(len(np.unique(y_train)))\n ]).to(device)\n \n # Use focal loss for better handling of imbalanced classes\n criterion = FocalLoss(alpha=1, gamma=2, weight=class_weights)\n \n # Optimizer with different learning rates\n optimizer = torch.optim.AdamW(\n model.parameters(),\n lr=1e-4,\n weight_decay=1e-4,\n betas=(0.9, 0.999)\n )\n \n # Learning rate scheduler\n scheduler = torch.optim.lr_scheduler.OneCycleLR(\n optimizer,\n max_lr=1e-3,\n epochs=200,\n steps_per_epoch=len(train_loader),\n pct_start=0.1,\n anneal_strategy='cos'\n )\n \n # Training parameters\n n_epochs = 200\n best_val_acc = 0\n patience = 12\n patience_counter = 0\n \n # Training history\n history = {\n 'train_loss': [], 'train_acc': [],\n 'val_loss': [], 'val_acc': []\n }\n \n print(\"Starting robust training...\")\n \n for epoch in range(n_epochs):\n # Training phase\n model.train()\n train_loss = 0\n train_correct = 0\n train_total = 0\n \n train_pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{n_epochs}\")\n \n for batch_idx, (data, target) in enumerate(train_pbar):\n try:\n data, target = data.to(device, non_blocking=True), target.to(device, non_blocking=True)\n \n optimizer.zero_grad()\n \n output = model(data)\n loss = criterion(output, target)\n \n loss.backward()\n \n # Gradient clipping\n torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n \n optimizer.step()\n scheduler.step()\n \n train_loss += loss.item()\n pred = output.argmax(dim=1)\n train_correct += pred.eq(target).sum().item()\n train_total += target.size(0)\n \n # Update progress bar\n current_acc = train_correct / train_total\n train_pbar.set_postfix({\n 'Loss': f'{loss.item():.4f}',\n 'Acc': f'{current_acc:.4f}',\n 'LR': f'{scheduler.get_last_lr()[0]:.2e}'\n })\n \n except RuntimeError as e:\n print(f\"Error in batch {batch_idx}: {e}\")\n continue\n \n # Validation phase\n model.eval()\n val_loss = 0\n val_correct = 0\n val_total = 0\n \n with torch.no_grad():\n for data, target in tqdm(val_loader, desc=\"Validation\"):\n data, target = data.to(device, non_blocking=True), target.to(device, non_blocking=True)\n output = model(data)\n \n val_loss += F.cross_entropy(output, target).item()\n pred = output.argmax(dim=1)\n val_correct += pred.eq(target).sum().item()\n val_total += target.size(0)\n \n # Calculate metrics\n train_loss = train_loss / len(train_loader)\n train_acc = train_correct / train_total\n val_loss = val_loss / len(val_loader)\n val_acc = val_correct / val_total\n \n # Save history\n history['train_loss'].append(train_loss)\n history['train_acc'].append(train_acc)\n history['val_loss'].append(val_loss)\n history['val_acc'].append(val_acc)\n \n # Print progress\n print(f'\\nEpoch {epoch+1}/{n_epochs} Results:')\n print(f' Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.4f}')\n print(f' Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.4f}')\n \n # Save best model\n if val_acc > best_val_acc:\n best_val_acc = val_acc\n torch.save({\n 'epoch': epoch,\n 'model_state_dict': model.state_dict(),\n 'optimizer_state_dict': optimizer.state_dict(),\n 'scheduler_state_dict': scheduler.state_dict(),\n 'best_val_acc': best_val_acc,\n 'history': history\n }, 'best_robust_model.pth')\n patience_counter = 0\n print(f' ✓ New best validation accuracy: {best_val_acc:.4f}')\n else:\n patience_counter += 1\n \n print('-' * 70)\n \n # Early stopping\n if patience_counter >= patience:\n print(f'Early stopping triggered at epoch {epoch+1}')\n break\n \n return model, history, best_val_acc\n\n# Execute the robust training\nprint(\"Starting robust training...\")\nmodel, history, best_acc = train_robust_model()\nprint(f\"Training completed! Best validation accuracy: {best_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T09:08:22.296867Z","iopub.execute_input":"2025-08-09T09:08:22.297449Z","iopub.status.idle":"2025-08-09T09:17:42.999364Z","shell.execute_reply.started":"2025-08-09T09:08:22.297422Z","shell.execute_reply":"2025-08-09T09:17:42.998444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 8.4: Test Time Augmentation (TTA) for better predictions\nclass TTAPredictor:\n def __init__(self, model, device, n_tta=5):\n self.model = model\n self.device = device\n self.n_tta = n_tta\n self.augment = AdvancedBehaviorDataset(np.array([]), np.array([]), augment=True)\n \n def predict_with_tta(self, data_loader):\n \"\"\"Predict with Test Time Augmentation\"\"\"\n self.model.eval()\n all_predictions = []\n all_probabilities = []\n \n with torch.no_grad():\n for data, _ in tqdm(data_loader, desc=\"TTA Prediction\"):\n batch_preds = []\n \n # Original prediction\n data_cuda = data.to(self.device)\n output = self.model(data_cuda)\n batch_preds.append(F.softmax(output, dim=1))\n \n # Augmented predictions\n for _ in range(self.n_tta):\n # Apply augmentations\n augmented_data = []\n for sample in data:\n aug_sample = self.augment.apply_augmentations(sample)\n augmented_data.append(aug_sample)\n \n augmented_batch = torch.stack(augmented_data).to(self.device)\n aug_output = self.model(augmented_batch)\n batch_preds.append(F.softmax(aug_output, dim=1))\n \n # Average predictions\n avg_pred = torch.stack(batch_preds).mean(dim=0)\n final_pred = avg_pred.argmax(dim=1)\n \n all_predictions.extend(final_pred.cpu().numpy())\n all_probabilities.extend(avg_pred.cpu().numpy())\n \n return np.array(all_predictions), np.array(all_probabilities)\n\n# Load best model and create TTA predictor\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ncheckpoint = torch.load('best_robust_model.pth', map_location=device)\nmodel.load_state_dict(checkpoint['model_state_dict'])\n\ntta_predictor = TTAPredictor(model, device, n_tta=3)\n\n# Create validation loader without augmentation\nval_dataset_clean = AdvancedBehaviorDataset(X_val, y_val, augment=False)\nval_loader_clean = DataLoader(val_dataset_clean, batch_size=32, shuffle=False)\n\n# Get TTA predictions\nprint(\"Generating predictions with Test Time Augmentation...\")\ntta_preds, tta_probs = tta_predictor.predict_with_tta(val_loader_clean)\n\n# Calculate TTA accuracy\ntta_accuracy = accuracy_score(y_val, tta_preds)\nprint(f\"Test Time Augmentation Accuracy: {tta_accuracy:.4f}\")\nprint(f\"Improvement over base model: {tta_accuracy - best_acc:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T09:19:01.025472Z","iopub.execute_input":"2025-08-09T09:19:01.026251Z","iopub.status.idle":"2025-08-09T09:19:02.681149Z","shell.execute_reply.started":"2025-08-09T09:19:01.026222Z","shell.execute_reply":"2025-08-09T09:19:02.680423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 1: Data Access and Environment Setup for CMI Detect Behavior Competition\n# This code should be run in a Kaggle notebook with GPU enabled\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.metrics import classification_report, confusion_matrix, f1_score, accuracy_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Set random seeds for reproducibility\ndef set_seed(seed=42):\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed_all(seed)\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n\nset_seed(42)\n\n# Check if GPU is available\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\n# Kaggle input paths\nINPUT_PATH = \"/kaggle/input/cmi-detect-behavior-with-sensor-data\"\n\n# Check available files\nprint(\"Available files in the dataset:\")\nfor root, dirs, files in os.walk(INPUT_PATH):\n level = root.replace(INPUT_PATH, '').count(os.sep)\n indent = ' ' * 4 * level\n print(f\"{indent}{os.path.basename(root)}/\")\n subindent = ' ' * 4 * (level + 1)\n for file in files:\n print(f\"{subindent}{file}\")\n\n# Load the training data\nprint(\"\\nLoading training data...\")\ntrain_data = []\ntrain_labels = []\n\n# Assuming the data structure based on common sensor data competition formats\n# Adjust paths based on actual dataset structure\ntry:\n # Load training data (adjust based on actual file structure)\n train_df = pd.read_csv(f\"{INPUT_PATH}/train.csv\")\n print(f\"Training data shape: {train_df.shape}\")\n print(f\"Training data columns: {train_df.columns.tolist()}\")\n print(train_df.head())\n \nexcept FileNotFoundError:\n print(\"train.csv not found. Checking for alternative file structures...\")\n # Check for parquet files or other formats\n files = os.listdir(INPUT_PATH)\n print(f\"Available files: {files}\")\n\n# Data loading function for sensor time series data\ndef load_sensor_data(data_path):\n \"\"\"\n Load sensor data from various file formats\n Returns processed time series data and labels\n \"\"\"\n data_files = []\n label_files = []\n \n # Check for different file formats\n for file in os.listdir(data_path):\n if file.endswith('.csv') and 'train' in file.lower():\n data_files.append(os.path.join(data_path, file))\n elif file.endswith('.parquet'):\n data_files.append(os.path.join(data_path, file))\n \n if not data_files:\n print(\"No recognized data files found. Please check the dataset structure.\")\n return None, None\n \n print(f\"Found data files: {data_files}\")\n return data_files\n\n# Load the data\ndata_files = load_sensor_data(INPUT_PATH)\n\nprint(\"Data access setup complete!\")\nprint(\"Next: Run the data preprocessing code...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T09:41:38.731899Z","iopub.execute_input":"2025-08-09T09:41:38.732753Z","iopub.status.idle":"2025-08-09T09:41:59.776044Z","shell.execute_reply.started":"2025-08-09T09:41:38.732721Z","shell.execute_reply":"2025-08-09T09:41:59.775232Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Data Preprocessing and Feature Engineering\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import signal\nfrom scipy.stats import entropy\nfrom sklearn.preprocessing import StandardScaler\nimport torch\n\nclass SensorDataPreprocessor:\n def __init__(self, window_size=128, overlap=0.5, target_freq=50):\n self.window_size = window_size\n self.overlap = overlap\n self.target_freq = target_freq\n self.scaler = StandardScaler()\n \n def load_and_preprocess_data(self, input_path):\n \"\"\"\n Load and preprocess sensor data from CMI competition\n Assumes typical sensor data format with accelerometer and gyroscope\n \"\"\"\n try:\n # Try different loading approaches based on file structure\n if os.path.exists(f\"{input_path}/train.csv\"):\n df = pd.read_csv(f\"{input_path}/train.csv\")\n elif os.path.exists(f\"{input_path}/series_train.parquet\"):\n df = pd.read_parquet(f\"{input_path}/series_train.parquet\")\n else:\n # Load all CSV files in the directory\n csv_files = [f for f in os.listdir(input_path) if f.endswith('.csv')]\n if csv_files:\n df = pd.read_csv(f\"{input_path}/{csv_files[0]}\")\n else:\n raise FileNotFoundError(\"No compatible data files found\")\n \n print(f\"Loaded data shape: {df.shape}\")\n print(f\"Columns: {df.columns.tolist()}\")\n print(df.head())\n \n return df\n \n except Exception as e:\n print(f\"Error loading data: {e}\")\n # Generate synthetic data for testing if real data not available\n return self._generate_synthetic_data()\n \n def _generate_synthetic_data(self):\n \"\"\"Generate synthetic sensor data for testing purposes\"\"\"\n print(\"Generating synthetic sensor data for testing...\")\n \n n_subjects = 100\n n_timesteps = 1000\n \n data = []\n for subject in range(n_subjects):\n # Generate realistic sensor patterns\n t = np.linspace(0, 10, n_timesteps)\n \n # Accelerometer data (3 axes)\n acc_x = np.sin(2 * np.pi * 1.5 * t) + 0.3 * np.random.normal(0, 1, n_timesteps)\n acc_y = np.cos(2 * np.pi * 1.2 * t) + 0.3 * np.random.normal(0, 1, n_timesteps)\n acc_z = 9.8 + 0.5 * np.sin(2 * np.pi * 0.8 * t) + 0.2 * np.random.normal(0, 1, n_timesteps)\n \n # Gyroscope data (3 axes)\n gyro_x = 0.1 * np.sin(2 * np.pi * 2.0 * t) + 0.05 * np.random.normal(0, 1, n_timesteps)\n gyro_y = 0.1 * np.cos(2 * np.pi * 1.8 * t) + 0.05 * np.random.normal(0, 1, n_timesteps)\n gyro_z = 0.05 * np.sin(2 * np.pi * 1.0 * t) + 0.03 * np.random.normal(0, 1, n_timesteps)\n \n # Label (0: no behavior, 1: behavior present)\n label = np.random.randint(0, 2)\n \n for i in range(n_timesteps):\n data.append({\n 'subject_id': subject,\n 'timestamp': i,\n 'acc_x': acc_x[i],\n 'acc_y': acc_y[i],\n 'acc_z': acc_z[i],\n 'gyro_x': gyro_x[i],\n 'gyro_y': gyro_y[i],\n 'gyro_z': gyro_z[i],\n 'label': label\n })\n \n return pd.DataFrame(data)\n \n def extract_features(self, signal_data):\n \"\"\"Extract time-domain and frequency-domain features\"\"\"\n features = {}\n \n # Time domain features\n features['mean'] = np.mean(signal_data)\n features['std'] = np.std(signal_data)\n features['min'] = np.min(signal_data)\n features['max'] = np.max(signal_data)\n features['range'] = features['max'] - features['min']\n features['rms'] = np.sqrt(np.mean(signal_data**2))\n features['skewness'] = pd.Series(signal_data).skew()\n features['kurtosis'] = pd.Series(signal_data).kurtosis()\n \n # Frequency domain features\n fft_vals = np.fft.fft(signal_data)\n fft_mag = np.abs(fft_vals)\n freqs = np.fft.fftfreq(len(signal_data))\n \n features['dominant_freq'] = freqs[np.argmax(fft_mag[1:len(fft_mag)//2]) + 1]\n features['spectral_energy'] = np.sum(fft_mag**2)\n features['spectral_entropy'] = entropy(fft_mag + 1e-12)\n \n return features\n \n def create_windows(self, df):\n \"\"\"Create sliding windows from time series data\"\"\"\n windows = []\n labels = []\n \n # Group by subject\n for subject_id in df['subject_id'].unique():\n subject_data = df[df['subject_id'] == subject_id].sort_values('timestamp')\n \n # Extract sensor columns\n sensor_cols = ['acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z']\n sensor_data = subject_data[sensor_cols].values\n \n if len(sensor_data) < self.window_size:\n continue\n \n # Create sliding windows\n step_size = int(self.window_size * (1 - self.overlap))\n for i in range(0, len(sensor_data) - self.window_size + 1, step_size):\n window = sensor_data[i:i + self.window_size]\n windows.append(window)\n labels.append(subject_data.iloc[i]['label'])\n \n return np.array(windows), np.array(labels)\n \n def apply_filters(self, data, low_freq=0.5, high_freq=20):\n \"\"\"Apply bandpass filter to sensor data\"\"\"\n filtered_data = np.zeros_like(data)\n \n for i in range(data.shape[-1]): # For each sensor channel\n # Apply Butterworth bandpass filter\n sos = signal.butter(4, [low_freq, high_freq], btype='band', \n fs=self.target_freq, output='sos')\n for j in range(data.shape[0]): # For each sample\n filtered_data[j, :, i] = signal.sosfilt(sos, data[j, :, i])\n \n return filtered_data\n \n def normalize_data(self, X_train, X_val=None, X_test=None):\n \"\"\"Normalize data using StandardScaler\"\"\"\n # Reshape for scaling\n n_samples, n_timesteps, n_features = X_train.shape\n X_train_reshaped = X_train.reshape(-1, n_features)\n \n # Fit scaler on training data\n X_train_scaled = self.scaler.fit_transform(X_train_reshaped)\n X_train_scaled = X_train_scaled.reshape(n_samples, n_timesteps, n_features)\n \n results = [X_train_scaled]\n \n # Transform validation data if provided\n if X_val is not None:\n n_val, _, _ = X_val.shape\n X_val_reshaped = X_val.reshape(-1, n_features)\n X_val_scaled = self.scaler.transform(X_val_reshaped)\n X_val_scaled = X_val_scaled.reshape(n_val, n_timesteps, n_features)\n results.append(X_val_scaled)\n \n # Transform test data if provided\n if X_test is not None:\n n_test, _, _ = X_test.shape\n X_test_reshaped = X_test.reshape(-1, n_features)\n X_test_scaled = self.scaler.transform(X_test_reshaped)\n X_test_scaled = X_test_scaled.reshape(n_test, n_timesteps, n_features)\n results.append(X_test_scaled)\n \n return results if len(results) > 1 else results[0]\n\n# Initialize preprocessor\npreprocessor = SensorDataPreprocessor(window_size=128, overlap=0.5)\n\n# Load and preprocess data\nprint(\"Loading data...\")\ndf = preprocessor.load_and_preprocess_data(INPUT_PATH)\n\n# Create windows\nprint(\"Creating time series windows...\")\nX, y = preprocessor.create_windows(df)\nprint(f\"Created {len(X)} windows with shape {X.shape}\")\nprint(f\"Labels distribution: {np.bincount(y)}\")\n\n# Split data\nX_train, X_temp, y_train, y_temp = train_test_split(\n X, y, test_size=0.3, random_state=42, stratify=y\n)\nX_val, X_test, y_val, y_test = train_test_split(\n X_temp, y_temp, test_size=0.5, random_state=42, stratify=y_temp\n)\n\n# Apply filters\nprint(\"Applying filters...\")\nX_train_filtered = preprocessor.apply_filters(X_train)\nX_val_filtered = preprocessor.apply_filters(X_val)\nX_test_filtered = preprocessor.apply_filters(X_test)\n\n# Normalize data\nprint(\"Normalizing data...\")\nX_train_norm, X_val_norm, X_test_norm = preprocessor.normalize_data(\n X_train_filtered, X_val_filtered, X_test_filtered\n)\n\nprint(f\"Training set shape: {X_train_norm.shape}\")\nprint(f\"Validation set shape: {X_val_norm.shape}\")\nprint(f\"Test set shape: {X_test_norm.shape}\")\n\n# Visualize data\nfig, axes = plt.subplots(2, 3, figsize=(15, 8))\nsensor_names = ['acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z']\n\nfor i, sensor in enumerate(sensor_names):\n ax = axes[i//3, i%3]\n \n # Plot sample from each class\n class_0_idx = np.where(y_train == 0)[0][0]\n class_1_idx = np.where(y_train == 1)[0][0]\n \n ax.plot(X_train_norm[class_0_idx, :, i], label='Class 0', alpha=0.7)\n ax.plot(X_train_norm[class_1_idx, :, i], label='Class 1', alpha=0.7)\n ax.set_title(f'{sensor}')\n ax.legend()\n ax.grid(True)\n\nplt.tight_layout()\nplt.savefig('sensor_data_visualization.png', dpi=300, bbox_inches='tight')\nplt.show()\n\nprint(\"Data preprocessing complete!\")\nprint(\"Next: Run the HMTAN model implementation...\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-09T09:42:16.164339Z","iopub.execute_input":"2025-08-09T09:42:16.164614Z","iopub.status.idle":"2025-08-09T09:42:35.491464Z","shell.execute_reply.started":"2025-08-09T09:42:16.164596Z","shell.execute_reply":"2025-08-09T09:42:35.49042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Data Loading and Initial Setup\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import StandardScaler, LabelEncoder\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import classification_report, confusion_matrix, accuracy_score\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Deep learning imports\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torch.nn.functional as F\n\n# Additional imports\nfrom scipy import stats\nfrom scipy.signal import butter, filtfilt\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n# Set random seeds for reproducibility\nnp.random.seed(42)\ntorch.manual_seed(42)\nif torch.cuda.is_available():\n torch.cuda.manual_seed(42)\n\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n\n# Load data\nprint(\"Loading training data...\")\ntrain_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv')\ntrain_demographics = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/train_demographics.csv')\n\nprint(\"Loading test data...\")\ntest_df = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv')\ntest_demographics = pd.read_csv('/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv')\n\nprint(f\"Train shape: {train_df.shape}\")\nprint(f\"Test shape: {test_df.shape}\")\nprint(f\"Train demographics shape: {train_demographics.shape}\")\nprint(f\"Test demographics shape: {test_demographics.shape}\")\n\n# Display basic info\nprint(\"\\nTraining data columns:\")\nprint(train_df.columns.tolist())\n\nprint(\"\\nUnique gestures in training data:\")\nprint(train_df['gesture'].unique())\n\nprint(\"\\nSequence types:\")\nprint(train_df['sequence_type'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T07:32:00.08617Z","iopub.execute_input":"2025-08-10T07:32:00.086352Z","iopub.status.idle":"2025-08-10T07:32:41.504523Z","shell.execute_reply.started":"2025-08-10T07:32:00.086334Z","shell.execute_reply":"2025-08-10T07:32:41.503713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Data Exploration and Preprocessing\ndef explore_data(train_df, demographics_df):\n \"\"\"Comprehensive data exploration\"\"\"\n \n print(\"=== DATA EXPLORATION ===\")\n \n # Basic statistics\n print(f\"Number of sequences: {train_df['sequence_id'].nunique()}\")\n print(f\"Number of subjects: {train_df['subject'].nunique()}\")\n print(f\"Average sequence length: {train_df.groupby('sequence_id').size().mean():.2f}\")\n \n # Behavior distribution\n print(f\"\\nBehavior distribution:\")\n behavior_counts = train_df['behavior'].value_counts()\n print(behavior_counts)\n \n # Gesture distribution\n print(f\"\\nGesture distribution:\")\n gesture_counts = train_df['gesture'].value_counts()\n print(gesture_counts)\n \n # Missing data analysis\n print(f\"\\nMissing data analysis:\")\n missing_data = train_df.isnull().sum()\n missing_cols = missing_data[missing_data > 0].sort_values(ascending=False)\n print(f\"Columns with missing data: {len(missing_cols)}\")\n if len(missing_cols) > 0:\n print(missing_cols.head(10))\n \n # Demographics exploration\n print(f\"\\nDemographics summary:\")\n print(demographics_df.describe())\n \n return behavior_counts, gesture_counts, missing_cols\n\n# Run exploration\nbehavior_counts, gesture_counts, missing_cols = explore_data(train_df, train_demographics)\n\n# Merge demographics with train data\ntrain_with_demo = train_df.merge(train_demographics, on='subject', how='left')\ntest_with_demo = test_df.merge(test_demographics, on='subject', how='left')\n\nprint(f\"\\nMerged train shape: {train_with_demo.shape}\")\nprint(f\"Merged test shape: {test_with_demo.shape}\")\n\n# Identify sensor columns\nacc_cols = [col for col in train_df.columns if col.startswith('acc_')]\nrot_cols = [col for col in train_df.columns if col.startswith('rot_')]\nthm_cols = [col for col in train_df.columns if col.startswith('thm_')]\ntof_cols = [col for col in train_df.columns if col.startswith('tof_')]\n\nprint(f\"\\nSensor columns:\")\nprint(f\"Accelerometer: {len(acc_cols)} columns\")\nprint(f\"Rotation: {len(rot_cols)} columns\") \nprint(f\"Thermopile: {len(thm_cols)} columns\")\nprint(f\"Time-of-flight: {len(tof_cols)} columns\")\n\n# Create target variable (BFRB vs non-BFRB)\ntarget_gestures = ['scratching_face', 'biting_fingernails', 'rubbing_body', \n 'adjusting_clothing', 'touching_hair', 'rubbing_face', \n 'picking_face', 'picking_fingernails']\n\ntrain_with_demo['is_bfrb'] = train_with_demo['gesture'].isin(target_gestures).astype(int)\n\nprint(f\"\\nBFRB distribution:\")\nprint(train_with_demo['is_bfrb'].value_counts())\nprint(f\"BFRB percentage: {train_with_demo['is_bfrb'].mean()*100:.2f}%\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T07:33:26.978007Z","iopub.execute_input":"2025-08-10T07:33:26.978569Z","iopub.status.idle":"2025-08-10T07:33:28.529205Z","shell.execute_reply.started":"2025-08-10T07:33:26.978537Z","shell.execute_reply":"2025-08-10T07:33:28.528497Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Advanced Feature Engineering\nclass SensorFeatureEngineer:\n \"\"\"Advanced feature engineering for sensor data\"\"\"\n \n def __init__(self, window_sizes=[5, 10, 20]):\n self.window_sizes = window_sizes\n self.scalers = {}\n \n def extract_statistical_features(self, df, sensor_cols):\n \"\"\"Extract statistical features for each sensor type\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns:\n # Basic statistics\n features[f'{col}_mean'] = df[col].mean()\n features[f'{col}_std'] = df[col].std()\n features[f'{col}_var'] = df[col].var()\n features[f'{col}_min'] = df[col].min()\n features[f'{col}_max'] = df[col].max()\n features[f'{col}_range'] = df[col].max() - df[col].min()\n features[f'{col}_median'] = df[col].median()\n features[f'{col}_q25'] = df[col].quantile(0.25)\n features[f'{col}_q75'] = df[col].quantile(0.75)\n features[f'{col}_iqr'] = df[col].quantile(0.75) - df[col].quantile(0.25)\n features[f'{col}_skew'] = df[col].skew()\n features[f'{col}_kurt'] = df[col].kurtosis()\n \n # Non-null ratio (for handling missing data)\n features[f'{col}_non_null_ratio'] = df[col].notna().mean()\n \n return features\n \n def extract_time_domain_features(self, df, sensor_cols):\n \"\"\"Extract time domain features\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns and df[col].notna().sum() > 1:\n values = df[col].fillna(method='ffill').fillna(method='bfill')\n \n # Zero crossing rate\n if len(values) > 1:\n features[f'{col}_zcr'] = np.sum(np.diff(np.sign(values)) != 0) / len(values)\n \n # Root mean square\n features[f'{col}_rms'] = np.sqrt(np.mean(values**2))\n \n # Energy\n features[f'{col}_energy'] = np.sum(values**2)\n \n # Mean absolute deviation\n features[f'{col}_mad'] = np.mean(np.abs(values - np.mean(values)))\n \n return features\n \n def extract_frequency_domain_features(self, df, sensor_cols, fs=50):\n \"\"\"Extract frequency domain features using FFT\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns and df[col].notna().sum() > 10:\n values = df[col].fillna(method='ffill').fillna(method='bfill')\n \n # FFT\n fft_vals = np.fft.fft(values)\n fft_magnitude = np.abs(fft_vals)\n freqs = np.fft.fftfreq(len(values), 1/fs)\n \n # Spectral features\n features[f'{col}_spectral_centroid'] = np.sum(freqs[:len(freqs)//2] * fft_magnitude[:len(freqs)//2]) / np.sum(fft_magnitude[:len(freqs)//2])\n features[f'{col}_spectral_rolloff'] = freqs[np.cumsum(fft_magnitude[:len(freqs)//2]) >= 0.85 * np.sum(fft_magnitude[:len(freqs)//2])][0] if np.sum(fft_magnitude[:len(freqs)//2]) > 0 else 0\n features[f'{col}_spectral_bandwidth'] = np.sqrt(np.sum(((freqs[:len(freqs)//2] - features[f'{col}_spectral_centroid'])**2) * fft_magnitude[:len(freqs)//2]) / np.sum(fft_magnitude[:len(freqs)//2]))\n \n # Dominant frequency\n dominant_freq_idx = np.argmax(fft_magnitude[1:len(freqs)//2]) + 1\n features[f'{col}_dominant_freq'] = freqs[dominant_freq_idx]\n \n return features\n \n def extract_motion_features(self, df):\n \"\"\"Extract motion-specific features from accelerometer and gyroscope data\"\"\"\n features = {}\n \n # Accelerometer magnitude\n if all(col in df.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n acc_mag = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n features['acc_magnitude_mean'] = acc_mag.mean()\n features['acc_magnitude_std'] = acc_mag.std()\n features['acc_magnitude_max'] = acc_mag.max()\n features['acc_magnitude_min'] = acc_mag.min()\n \n # Jerk (rate of change of acceleration)\n acc_jerk = np.sqrt(np.diff(df['acc_x'])**2 + np.diff(df['acc_y'])**2 + np.diff(df['acc_z'])**2)\n if len(acc_jerk) > 0:\n features['acc_jerk_mean'] = acc_jerk.mean()\n features['acc_jerk_std'] = acc_jerk.std()\n \n # Rotation magnitude\n if all(col in df.columns for col in ['rot_x', 'rot_y', 'rot_z']):\n rot_mag = np.sqrt(df['rot_x']**2 + df['rot_y']**2 + df['rot_z']**2)\n features['rot_magnitude_mean'] = rot_mag.mean()\n features['rot_magnitude_std'] = rot_mag.std()\n features['rot_magnitude_max'] = rot_mag.max()\n features['rot_magnitude_min'] = rot_mag.min()\n \n return features\n \n def extract_sequence_features(self, sequence_df, sensor_types):\n \"\"\"Extract all features for a single sequence\"\"\"\n all_features = {}\n \n # Statistical features for each sensor type\n for sensor_type, cols in sensor_types.items():\n stat_features = self.extract_statistical_features(sequence_df, cols)\n all_features.update(stat_features)\n \n time_features = self.extract_time_domain_features(sequence_df, cols)\n all_features.update(time_features)\n \n if sensor_type in ['acc', 'rot']: # Only for high-frequency sensors\n freq_features = self.extract_frequency_domain_features(sequence_df, cols)\n all_features.update(freq_features)\n \n # Motion-specific features\n motion_features = self.extract_motion_features(sequence_df)\n all_features.update(motion_features)\n \n # Sequence-level features\n all_features['sequence_length'] = len(sequence_df)\n all_features['transition_duration'] = len(sequence_df[sequence_df['behavior'] == 'Transition'])\n all_features['pause_duration'] = len(sequence_df[sequence_df['behavior'] == 'Pause'])\n all_features['gesture_duration'] = len(sequence_df[sequence_df['behavior'] == 'Gesture'])\n \n return all_features\n\n# Initialize feature engineer\nfe = SensorFeatureEngineer()\n\n# Define sensor types\nsensor_types = {\n 'acc': acc_cols,\n 'rot': rot_cols,\n 'thm': thm_cols,\n 'tof': tof_cols\n}\n\nprint(\"Feature engineering setup complete!\")\nprint(f\"Sensor types: {list(sensor_types.keys())}\")\nprint(f\"Total sensor columns: {sum(len(cols) for cols in sensor_types.values())}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T07:34:17.229354Z","iopub.execute_input":"2025-08-10T07:34:17.229926Z","iopub.status.idle":"2025-08-10T07:34:17.249848Z","shell.execute_reply.started":"2025-08-10T07:34:17.229901Z","shell.execute_reply":"2025-08-10T07:34:17.249026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Advanced Feature Engineering\nclass SensorFeatureEngineer:\n \"\"\"Advanced feature engineering for sensor data\"\"\"\n \n def __init__(self, window_sizes=[5, 10, 20]):\n self.window_sizes = window_sizes\n self.scalers = {}\n \n def extract_statistical_features(self, df, sensor_cols):\n \"\"\"Extract statistical features for each sensor type\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns:\n # Basic statistics\n features[f'{col}_mean'] = df[col].mean()\n features[f'{col}_std'] = df[col].std()\n features[f'{col}_var'] = df[col].var()\n features[f'{col}_min'] = df[col].min()\n features[f'{col}_max'] = df[col].max()\n features[f'{col}_range'] = df[col].max() - df[col].min()\n features[f'{col}_median'] = df[col].median()\n features[f'{col}_q25'] = df[col].quantile(0.25)\n features[f'{col}_q75'] = df[col].quantile(0.75)\n features[f'{col}_iqr'] = df[col].quantile(0.75) - df[col].quantile(0.25)\n features[f'{col}_skew'] = df[col].skew()\n features[f'{col}_kurt'] = df[col].kurtosis()\n \n # Non-null ratio (for handling missing data)\n features[f'{col}_non_null_ratio'] = df[col].notna().mean()\n \n return features\n \n def extract_time_domain_features(self, df, sensor_cols):\n \"\"\"Extract time domain features\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns and df[col].notna().sum() > 1:\n values = df[col].fillna(method='ffill').fillna(method='bfill')\n \n # Zero crossing rate\n if len(values) > 1:\n features[f'{col}_zcr'] = np.sum(np.diff(np.sign(values)) != 0) / len(values)\n \n # Root mean square\n features[f'{col}_rms'] = np.sqrt(np.mean(values**2))\n \n # Energy\n features[f'{col}_energy'] = np.sum(values**2)\n \n # Mean absolute deviation\n features[f'{col}_mad'] = np.mean(np.abs(values - np.mean(values)))\n \n return features\n \n def extract_frequency_domain_features(self, df, sensor_cols, fs=50):\n \"\"\"Extract frequency domain features using FFT\"\"\"\n features = {}\n \n for col in sensor_cols:\n if col in df.columns and df[col].notna().sum() > 10:\n values = df[col].fillna(method='ffill').fillna(method='bfill')\n \n # Skip if values are all the same or empty\n if len(values) == 0 or values.var() == 0:\n features[f'{col}_spectral_centroid'] = 0\n features[f'{col}_spectral_rolloff'] = 0\n features[f'{col}_spectral_bandwidth'] = 0\n features[f'{col}_dominant_freq'] = 0\n continue\n \n try:\n # FFT\n fft_vals = np.fft.fft(values)\n fft_magnitude = np.abs(fft_vals)\n freqs = np.fft.fftfreq(len(values), 1/fs)\n \n # Only use positive frequencies\n pos_freqs = freqs[:len(freqs)//2]\n pos_magnitude = fft_magnitude[:len(freqs)//2]\n \n # Avoid division by zero\n total_magnitude = np.sum(pos_magnitude)\n if total_magnitude == 0:\n features[f'{col}_spectral_centroid'] = 0\n features[f'{col}_spectral_rolloff'] = 0\n features[f'{col}_spectral_bandwidth'] = 0\n features[f'{col}_dominant_freq'] = 0\n continue\n \n # Spectral centroid\n spectral_centroid = np.sum(pos_freqs * pos_magnitude) / total_magnitude\n features[f'{col}_spectral_centroid'] = spectral_centroid\n \n # Spectral rolloff (frequency below which 85% of energy is contained)\n cumsum_magnitude = np.cumsum(pos_magnitude)\n rolloff_threshold = 0.85 * total_magnitude\n rolloff_indices = np.where(cumsum_magnitude >= rolloff_threshold)[0]\n \n if len(rolloff_indices) > 0:\n features[f'{col}_spectral_rolloff'] = pos_freqs[rolloff_indices[0]]\n else:\n features[f'{col}_spectral_rolloff'] = pos_freqs[-1] if len(pos_freqs) > 0 else 0\n \n # Spectral bandwidth\n spectral_bandwidth = np.sqrt(np.sum(((pos_freqs - spectral_centroid)**2) * pos_magnitude) / total_magnitude)\n features[f'{col}_spectral_bandwidth'] = spectral_bandwidth\n \n # Dominant frequency\n if len(pos_magnitude) > 1:\n dominant_freq_idx = np.argmax(pos_magnitude[1:]) + 1 # Skip DC component\n features[f'{col}_dominant_freq'] = pos_freqs[dominant_freq_idx] if dominant_freq_idx < len(pos_freqs) else 0\n else:\n features[f'{col}_dominant_freq'] = 0\n \n except Exception as e:\n # If any error occurs, set default values\n features[f'{col}_spectral_centroid'] = 0\n features[f'{col}_spectral_rolloff'] = 0\n features[f'{col}_spectral_bandwidth'] = 0\n features[f'{col}_dominant_freq'] = 0\n \n return features\n \n def extract_motion_features(self, df):\n \"\"\"Extract motion-specific features from accelerometer and gyroscope data\"\"\"\n features = {}\n \n # Accelerometer magnitude\n if all(col in df.columns for col in ['acc_x', 'acc_y', 'acc_z']):\n acc_mag = np.sqrt(df['acc_x']**2 + df['acc_y']**2 + df['acc_z']**2)\n features['acc_magnitude_mean'] = acc_mag.mean()\n features['acc_magnitude_std'] = acc_mag.std()\n features['acc_magnitude_max'] = acc_mag.max()\n features['acc_magnitude_min'] = acc_mag.min()\n \n # Jerk (rate of change of acceleration)\n acc_jerk = np.sqrt(np.diff(df['acc_x'])**2 + np.diff(df['acc_y'])**2 + np.diff(df['acc_z'])**2)\n if len(acc_jerk) > 0:\n features['acc_jerk_mean'] = acc_jerk.mean()\n features['acc_jerk_std'] = acc_jerk.std()\n \n # Rotation magnitude\n if all(col in df.columns for col in ['rot_x', 'rot_y', 'rot_z']):\n rot_mag = np.sqrt(df['rot_x']**2 + df['rot_y']**2 + df['rot_z']**2)\n features['rot_magnitude_mean'] = rot_mag.mean()\n features['rot_magnitude_std'] = rot_mag.std()\n features['rot_magnitude_max'] = rot_mag.max()\n features['rot_magnitude_min'] = rot_mag.min()\n \n return features\n \n def extract_sequence_features(self, sequence_df, sensor_types):\n \"\"\"Extract all features for a single sequence\"\"\"\n all_features = {}\n \n # Statistical features for each sensor type\n for sensor_type, cols in sensor_types.items():\n stat_features = self.extract_statistical_features(sequence_df, cols)\n all_features.update(stat_features)\n \n time_features = self.extract_time_domain_features(sequence_df, cols)\n all_features.update(time_features)\n \n if sensor_type in ['acc', 'rot']: # Only for high-frequency sensors\n freq_features = self.extract_frequency_domain_features(sequence_df, cols)\n all_features.update(freq_features)\n \n # Motion-specific features\n motion_features = self.extract_motion_features(sequence_df)\n all_features.update(motion_features)\n \n # Sequence-level features\n all_features['sequence_length'] = len(sequence_df)\n all_features['transition_duration'] = len(sequence_df[sequence_df['behavior'] == 'Transition'])\n all_features['pause_duration'] = len(sequence_df[sequence_df['behavior'] == 'Pause'])\n all_features['gesture_duration'] = len(sequence_df[sequence_df['behavior'] == 'Gesture'])\n \n return all_features\n\n# Initialize feature engineer\nfe = SensorFeatureEngineer()\n\n# Define sensor types\nsensor_types = {\n 'acc': acc_cols,\n 'rot': rot_cols,\n 'thm': thm_cols,\n 'tof': tof_cols\n}\n\nprint(\"Feature engineering setup complete!\")\nprint(f\"Sensor types: {list(sensor_types.keys())}\")\nprint(f\"Total sensor columns: {sum(len(cols) for cols in sensor_types.values())}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T07:38:49.501375Z","iopub.execute_input":"2025-08-10T07:38:49.501931Z","iopub.status.idle":"2025-08-10T07:38:49.523095Z","shell.execute_reply.started":"2025-08-10T07:38:49.501907Z","shell.execute_reply":"2025-08-10T07:38:49.522447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Feature Extraction Process\ndef extract_features_for_dataset(df, demographics_df, feature_engineer, sensor_types, is_train=True):\n \"\"\"Extract features for entire dataset\"\"\"\n \n # Merge with demographics\n df_with_demo = df.merge(demographics_df, on='subject', how='left')\n \n sequence_features = []\n sequence_ids = []\n targets = []\n \n unique_sequences = df_with_demo['sequence_id'].unique()\n \n print(f\"Processing {len(unique_sequences)} sequences...\")\n \n for i, seq_id in enumerate(unique_sequences):\n if i % 100 == 0:\n print(f\"Processed {i}/{len(unique_sequences)} sequences\")\n \n # Get sequence data\n seq_data = df_with_demo[df_with_demo['sequence_id'] == seq_id].copy()\n \n # Extract sensor features\n sensor_features = feature_engineer.extract_sequence_features(seq_data, sensor_types)\n \n # Add demographic features\n demo_features = {\n 'adult_child': seq_data['adult_child'].iloc[0],\n 'age': seq_data['age'].iloc[0],\n 'sex': seq_data['sex'].iloc[0],\n 'handedness': seq_data['handedness'].iloc[0],\n 'height_cm': seq_data['height_cm'].iloc[0],\n 'shoulder_to_wrist_cm': seq_data['shoulder_to_wrist_cm'].iloc[0],\n 'elbow_to_wrist_cm': seq_data['elbow_to_wrist_cm'].iloc[0],\n }\n \n # Combine all features\n all_features = {**sensor_features, **demo_features}\n \n sequence_features.append(all_features)\n sequence_ids.append(seq_id)\n \n if is_train:\n # Create target - check if this is a BFRB gesture\n gesture = seq_data['gesture'].iloc[0]\n target_gestures = ['scratching_face', 'biting_fingernails', 'rubbing_body', \n 'adjusting_clothing', 'touching_hair', 'rubbing_face', \n 'picking_face', 'picking_fingernails']\n is_bfrb = 1 if gesture in target_gestures else 0\n targets.append(is_bfrb)\n \n # Convert to DataFrame\n features_df = pd.DataFrame(sequence_features, index=sequence_ids)\n \n # Handle missing values\n features_df = features_df.fillna(features_df.median())\n \n # Handle infinite values\n features_df = features_df.replace([np.inf, -np.inf], np.nan)\n features_df = features_df.fillna(features_df.median())\n \n print(f\"Extracted {len(features_df.columns)} features\")\n print(f\"Feature matrix shape: {features_df.shape}\")\n \n if is_train:\n return features_df, np.array(targets)\n else:\n return features_df\n\n# Extract features for training data (this may take a few minutes)\nprint(\"Extracting training features...\")\nX_train_features, y_train = extract_features_for_dataset(\n train_df, train_demographics, fe, sensor_types, is_train=True\n)\n\nprint(f\"\\nTraining features shape: {X_train_features.shape}\")\nprint(f\"Training target shape: {y_train.shape}\")\nprint(f\"BFRB distribution: {np.bincount(y_train)} (ratio: {y_train.mean():.3f})\")\n\n# Display some feature statistics\nprint(f\"\\nFeature statistics:\")\nprint(f\"Number of features: {len(X_train_features.columns)}\")\nprint(f\"Features with all zeros: {(X_train_features == 0).all().sum()}\")\nprint(f\"Features with missing values: {X_train_features.isnull().sum().sum()}\")\n\n# Show sample of features\nprint(f\"\\nSample features:\")\nprint(X_train_features.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T07:40:13.892277Z","iopub.execute_input":"2025-08-10T07:40:13.892863Z","iopub.status.idle":"2025-08-10T09:49:26.299929Z","shell.execute_reply.started":"2025-08-10T07:40:13.892838Z","shell.execute_reply":"2025-08-10T09:49:26.299315Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Deep Learning - Sequence Dataset and Models\nclass SequenceDataset(Dataset):\n \"\"\"Dataset for sequence data with padding\"\"\"\n \n def __init__(self, sequences, targets, sensor_cols, demo_features=None, max_length=None):\n self.sequences = sequences\n self.targets = targets\n self.sensor_cols = sensor_cols\n self.demo_features = demo_features\n self.max_length = max_length\n \n if max_length is None:\n self.max_length = max([len(seq) for seq in sequences])\n \n def __len__(self):\n return len(self.sequences)\n \n def __getitem__(self, idx):\n # Get sequence data\n sequence = self.sequences[idx][self.sensor_cols].values\n target = self.targets[idx]\n \n # Pad or truncate sequence\n if len(sequence) < self.max_length:\n # Pad with zeros\n padded_sequence = np.zeros((self.max_length, len(self.sensor_cols)))\n padded_sequence[:len(sequence)] = sequence\n sequence = padded_sequence\n elif len(sequence) > self.max_length:\n # Truncate\n sequence = sequence[:self.max_length]\n \n # Add demographic features if available\n if self.demo_features is not None:\n demo_data = self.demo_features[idx]\n return torch.FloatTensor(sequence), torch.FloatTensor(demo_data), torch.LongTensor([target])\n \n return torch.FloatTensor(sequence), torch.LongTensor([target])\n\nclass MultiModalLSTM(nn.Module):\n \"\"\"Multi-modal LSTM for sensor and demographic data\"\"\"\n \n def __init__(self, sensor_input_size, demo_input_size=0, hidden_size=128, \n num_layers=2, num_classes=2, dropout=0.3):\n super(MultiModalLSTM, self).__init__()\n \n self.sensor_input_size = sensor_input_size\n self.demo_input_size = demo_input_size\n self.hidden_size = hidden_size\n self.num_layers = num_layers\n self.dropout = dropout\n \n # Sensor data processing\n self.lstm = nn.LSTM(sensor_input_size, hidden_size, num_layers, \n batch_first=True, dropout=dropout if num_layers > 1 else 0)\n \n # Attention mechanism\n self.attention = nn.MultiheadAttention(hidden_size, num_heads=8, dropout=dropout)\n \n # Demographic data processing\n if demo_input_size > 0:\n self.demo_fc = nn.Sequential(\n nn.Linear(demo_input_size, hidden_size // 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(hidden_size // 2, hidden_size // 4)\n )\n final_size = hidden_size + hidden_size // 4\n else:\n final_size = hidden_size\n \n # Final classification layers\n self.classifier = nn.Sequential(\n nn.Linear(final_size, hidden_size // 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(hidden_size // 2, hidden_size // 4),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(hidden_size // 4, num_classes)\n )\n \n def forward(self, sensor_data, demo_data=None):\n batch_size = sensor_data.size(0)\n \n # LSTM forward pass\n lstm_out, (hidden, cell) = self.lstm(sensor_data)\n \n # Apply attention\n lstm_out_transposed = lstm_out.transpose(0, 1) # (seq, batch, hidden)\n attended_out, _ = self.attention(lstm_out_transposed, lstm_out_transposed, lstm_out_transposed)\n attended_out = attended_out.transpose(0, 1) # (batch, seq, hidden)\n \n # Global average pooling\n sensor_features = torch.mean(attended_out, dim=1) # (batch, hidden)\n \n # Combine with demographic data if available\n if demo_data is not None and self.demo_input_size > 0:\n demo_features = self.demo_fc(demo_data)\n combined_features = torch.cat([sensor_features, demo_features], dim=1)\n else:\n combined_features = sensor_features\n \n # Final classification\n output = self.classifier(combined_features)\n \n return output\n\nclass TransformerModel(nn.Module):\n \"\"\"Transformer model for sequence classification\"\"\"\n \n def __init__(self, input_size, d_model=128, nhead=8, num_layers=4, \n num_classes=2, dropout=0.1, max_length=1000):\n super(TransformerModel, self).__init__()\n \n self.d_model = d_model\n self.input_projection = nn.Linear(input_size, d_model)\n self.positional_encoding = nn.Parameter(torch.randn(max_length, d_model))\n \n encoder_layer = nn.TransformerEncoderLayer(\n d_model=d_model, nhead=nhead, dropout=dropout, batch_first=True\n )\n self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n \n self.classifier = nn.Sequential(\n nn.Linear(d_model, d_model // 2),\n nn.ReLU(),\n nn.Dropout(dropout),\n nn.Linear(d_model // 2, num_classes)\n )\n \n def forward(self, x):\n seq_len = x.size(1)\n \n # Project input to d_model dimensions\n x = self.input_projection(x)\n \n # Add positional encoding\n x = x + self.positional_encoding[:seq_len].unsqueeze(0)\n \n # Transformer encoding\n x = self.transformer(x)\n \n # Global average pooling\n x = torch.mean(x, dim=1)\n \n # Classification\n output = self.classifier(x)\n \n return output\n\nprint(\"Deep learning models defined successfully!\")\nprint(\"Available models:\")\nprint(\"1. MultiModalLSTM - LSTM with attention and demographic fusion\")\nprint(\"2. TransformerModel - Transformer encoder for sequences\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T09:50:39.813422Z","iopub.execute_input":"2025-08-10T09:50:39.814229Z","iopub.status.idle":"2025-08-10T09:50:39.829546Z","shell.execute_reply.started":"2025-08-10T09:50:39.814089Z","shell.execute_reply":"2025-08-10T09:50:39.828927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Sequence Data Preparation for Deep Learning\ndef prepare_sequence_data(df, demographics_df, sensor_types, is_train=True):\n \"\"\"Prepare sequence data for deep learning models\"\"\"\n \n # Merge with demographics\n df_with_demo = df.merge(demographics_df, on='subject', how='left')\n \n sequences = []\n targets = []\n demo_features = []\n sequence_ids = []\n \n # Get only sensor columns (exclude metadata)\n sensor_cols = []\n for sensor_type, cols in sensor_types.items():\n sensor_cols.extend(cols)\n \n # Remove columns that don't exist in the dataframe\n sensor_cols = [col for col in sensor_cols if col in df_with_demo.columns]\n \n unique_sequences = df_with_demo['sequence_id'].unique()\n \n print(f\"Preparing {len(unique_sequences)} sequences...\")\n print(f\"Using {len(sensor_cols)} sensor features\")\n \n for i, seq_id in enumerate(unique_sequences):\n if i % 100 == 0:\n print(f\"Processed {i}/{len(unique_sequences)} sequences\")\n \n # Get sequence data\n seq_data = df_with_demo[df_with_demo['sequence_id'] == seq_id].copy()\n \n # Get sensor data\n sensor_sequence = seq_data[sensor_cols].fillna(0).values\n \n # Get demographic features\n demo_row = seq_data.iloc[0]\n demo_vector = [\n demo_row['adult_child'],\n demo_row['age'] / 100.0, # Normalize age\n demo_row['sex'],\n demo_row['handedness'],\n demo_row['height_cm'] / 200.0, # Normalize height\n demo_row['shoulder_to_wrist_cm'] / 100.0, # Normalize\n demo_row['elbow_to_wrist_cm'] / 100.0 # Normalize\n ]\n \n sequences.append(sensor_sequence)\n demo_features.append(demo_vector)\n sequence_ids.append(seq_id)\n \n if is_train:\n # Create target\n gesture = seq_data['gesture'].iloc[0]\n target_gestures = ['scratching_face', 'biting_fingernails', 'rubbing_body', \n 'adjusting_clothing', 'touching_hair', 'rubbing_face', \n 'picking_face', 'picking_fingernails']\n is_bfrb = 1 if gesture in target_gestures else 0\n targets.append(is_bfrb)\n \n # Convert to numpy arrays\n demo_features = np.array(demo_features)\n \n # Handle missing values in demographic features\n for i in range(demo_features.shape[1]):\n col_median = np.nanmedian(demo_features[:, i])\n demo_features[:, i] = np.where(np.isnan(demo_features[:, i]), col_median, demo_features[:, i])\n \n if is_train:\n targets = np.array(targets)\n return sequences, targets, demo_features, sensor_cols, sequence_ids\n else:\n return sequences, demo_features, sensor_cols, sequence_ids\n\n# Prepare sequence data for deep learning\nprint(\"Preparing sequence data for deep learning...\")\nsequences, targets, demo_features, sensor_cols, seq_ids = prepare_sequence_data(\n train_df, train_demographics, sensor_types, is_train=True\n)\n\nprint(f\"Number of sequences: {len(sequences)}\")\nprint(f\"Sequence lengths: min={min(len(s) for s in sequences)}, max={max(len(s) for s in sequences)}, avg={np.mean([len(s) for s in sequences]):.1f}\")\nprint(f\"Number of sensor features: {len(sensor_cols)}\")\nprint(f\"Demo features shape: {demo_features.shape}\")\nprint(f\"Target distribution: {np.bincount(targets)}\")\n\n# Set maximum sequence length (for computational efficiency)\nmax_seq_length = min(1000, max(len(s) for s in sequences))\nprint(f\"Using max sequence length: {max_seq_length}\")\n\n# Split data for training\nfrom sklearn.model_selection import train_test_split\n\n# Split at sequence level to avoid data leakage\ntrain_sequences, val_sequences, train_targets, val_targets, train_demo, val_demo = train_test_split(\n sequences, targets, demo_features, test_size=0.2, random_state=42, stratify=targets\n)\n\nprint(f\"Train sequences: {len(train_sequences)}\")\nprint(f\"Validation sequences: {len(val_sequences)}\")\nprint(f\"Train target distribution: {np.bincount(train_targets)}\")\nprint(f\"Val target distribution: {np.bincount(val_targets)}\")\n\n# Create datasets\ntrain_dataset = SequenceDataset(\n [pd.DataFrame(seq, columns=sensor_cols) for seq in train_sequences],\n train_targets, \n sensor_cols, \n demo_features=train_demo,\n max_length=max_seq_length\n)\n\nval_dataset = SequenceDataset(\n [pd.DataFrame(seq, columns=sensor_cols) for seq in val_sequences],\n val_targets, \n sensor_cols, \n demo_features=val_demo,\n max_length=max_seq_length\n)\n\nprint(\"Sequence datasets created successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T09:52:27.653866Z","iopub.execute_input":"2025-08-10T09:52:27.654387Z","iopub.status.idle":"2025-08-10T09:57:47.437646Z","shell.execute_reply.started":"2025-08-10T09:52:27.654362Z","shell.execute_reply":"2025-08-10T09:57:47.436853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Traditional ML Models with Advanced Techniques\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import classification_report, roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import cross_val_score, StratifiedKFold\nfrom sklearn.feature_selection import SelectKBest, f_classif, RFE\nfrom sklearn.preprocessing import StandardScaler\nimport xgboost as xgb\nimport lightgbm as lgb\nfrom sklearn.calibration import CalibratedClassifierCV\n\nclass AdvancedMLPipeline:\n \"\"\"Advanced ML pipeline with feature selection and ensemble methods\"\"\"\n \n def __init__(self):\n self.scaler = StandardScaler()\n self.feature_selector = None\n self.models = {}\n self.ensemble = None\n \n def prepare_features(self, X_train, y_train, X_val=None, feature_selection_k=500):\n \"\"\"Feature scaling and selection\"\"\"\n \n # Scale features\n X_train_scaled = self.scaler.fit_transform(X_train)\n if X_val is not None:\n X_val_scaled = self.scaler.transform(X_val)\n \n # Feature selection\n print(f\"Original features: {X_train.shape[1]}\")\n \n # Remove constant features\n constant_features = X_train.columns[X_train.var() == 0]\n if len(constant_features) > 0:\n print(f\"Removing {len(constant_features)} constant features\")\n X_train = X_train.drop(columns=constant_features)\n if X_val is not None:\n X_val = X_val.drop(columns=constant_features)\n \n # Select best features\n self.feature_selector = SelectKBest(f_classif, k=min(feature_selection_k, X_train.shape[1]))\n X_train_selected = self.feature_selector.fit_transform(X_train, y_train)\n \n selected_features = X_train.columns[self.feature_selector.get_support()]\n print(f\"Selected features: {len(selected_features)}\")\n \n if X_val is not None:\n X_val_selected = self.feature_selector.transform(X_val)\n return X_train_selected, X_val_selected, selected_features\n \n return X_train_selected, selected_features\n \n def check_class_balance(self, y):\n \"\"\"Check and report class balance\"\"\"\n unique, counts = np.unique(y, return_counts=True)\n class_dist = dict(zip(unique, counts))\n \n print(f\"Class distribution: {class_dist}\")\n \n # Check if we have at least 2 classes with reasonable samples\n if len(unique) < 2:\n raise ValueError(f\"Only {len(unique)} class found in data. Need at least 2 classes for classification.\")\n \n min_samples = min(counts)\n if min_samples < 5:\n print(f\"Warning: Minority class has only {min_samples} samples. This may cause training issues.\")\n \n return class_dist\n \n def train_individual_models(self, X_train, y_train):\n \"\"\"Train individual models\"\"\"\n \n print(\"Training individual models...\")\n \n # Check class balance first\n class_dist = self.check_class_balance(y_train)\n \n # Calculate class weights safely\n n_positive = len(y_train[y_train == 1])\n n_negative = len(y_train[y_train == 0])\n \n # Avoid division by zero\n if n_positive == 0:\n scale_pos_weight = 1.0\n print(\"Warning: No positive samples found. Using scale_pos_weight=1.0\")\n else:\n scale_pos_weight = n_negative / n_positive\n \n print(f\"Scale pos weight: {scale_pos_weight:.3f}\")\n \n # Define models with more conservative parameters\n models = {\n 'random_forest': RandomForestClassifier(\n n_estimators=100, max_depth=10, min_samples_split=10,\n min_samples_leaf=5, random_state=42, n_jobs=-1,\n class_weight='balanced'\n ),\n 'xgboost': xgb.XGBClassifier(\n n_estimators=100, max_depth=6, learning_rate=0.1,\n subsample=0.8, colsample_bytree=0.8, random_state=42,\n eval_metric='logloss', scale_pos_weight=scale_pos_weight\n ),\n 'lightgbm': lgb.LGBMClassifier(\n n_estimators=100, max_depth=6, learning_rate=0.1,\n num_leaves=31, subsample=0.8, colsample_bytree=0.8,\n random_state=42, class_weight='balanced', verbosity=-1\n ),\n 'logistic_regression': LogisticRegression(\n random_state=42, max_iter=1000, class_weight='balanced',\n C=1.0\n )\n }\n \n # Train and evaluate each model\n cv_scores = {}\n \n # Use simpler cross-validation if class imbalance is severe\n min_class_samples = min(class_dist.values())\n if min_class_samples < 10:\n print(f\"Using simple train-validation split due to severe class imbalance (min class: {min_class_samples} samples)\")\n \n # Simple validation without cross-validation\n for name, model in models.items():\n print(f\"Training {name}...\")\n try:\n # Train on full training set\n model.fit(X_train, y_train)\n \n # Simple accuracy score\n train_score = model.score(X_train, y_train)\n cv_scores[name] = np.array([train_score])\n \n print(f\"{name} Training Accuracy: {train_score:.4f}\")\n \n self.models[name] = model\n \n except Exception as e:\n print(f\"Failed to train {name}: {str(e)}\")\n continue\n else:\n # Use cross-validation for better balanced data\n skf = StratifiedKFold(n_splits=min(5, min_class_samples//2), shuffle=True, random_state=42)\n \n for name, model in models.items():\n print(f\"Training {name}...\")\n try:\n # Cross-validation with error handling\n cv_score = cross_val_score(\n model, X_train, y_train, cv=skf, \n scoring='accuracy', n_jobs=1, error_score='raise'\n )\n cv_scores[name] = cv_score\n \n print(f\"{name} CV Accuracy: {cv_score.mean():.4f} (+/- {cv_score.std() * 2:.4f})\")\n \n # Train on full training set\n model.fit(X_train, y_train)\n self.models[name] = model\n \n except Exception as e:\n print(f\"Failed to train {name}: {str(e)}\")\n continue\n \n return cv_scores\n \n def create_ensemble(self, X_train, y_train):\n \"\"\"Create ensemble model\"\"\"\n \n if len(self.models) < 2:\n print(\"Not enough models for ensemble. Skipping ensemble creation.\")\n return\n \n # Select best performing models for ensemble\n best_models = []\n for name, model in self.models.items():\n if name in ['random_forest', 'xgboost', 'lightgbm']:\n best_models.append((name[:3], model))\n \n if len(best_models) < 2:\n best_models = [(name[:3], model) for name, model in list(self.models.items())[:3]]\n \n try:\n # Create voting ensemble\n self.ensemble = VotingClassifier(\n estimators=best_models,\n voting='hard' # Use hard voting for stability\n )\n \n # Train ensemble\n self.ensemble.fit(X_train, y_train)\n \n print(\"Ensemble model created successfully\")\n \n except Exception as e:\n print(f\"Failed to create ensemble: {str(e)}\")\n self.ensemble = None\n \n def evaluate_models(self, X_val, y_val):\n \"\"\"Evaluate all models\"\"\"\n \n results = {}\n \n for name, model in self.models.items():\n try:\n y_pred = model.predict(X_val)\n \n # Calculate metrics\n accuracy = accuracy_score(y_val, y_pred)\n \n # Only calculate ROC-AUC if we have probability predictions and both classes\n try:\n if hasattr(model, 'predict_proba') and len(np.unique(y_val)) > 1:\n y_prob = model.predict_proba(X_val)[:, 1]\n roc_auc = roc_auc_score(y_val, y_prob)\n else:\n roc_auc = 0.5 # Default value\n except:\n roc_auc = 0.5\n \n results[name] = {\n 'accuracy': accuracy,\n 'roc_auc': roc_auc,\n 'classification_report': classification_report(y_val, y_pred)\n }\n \n except Exception as e:\n print(f\"Failed to evaluate {name}: {str(e)}\")\n continue\n \n # Evaluate ensemble if available\n if self.ensemble is not None:\n try:\n y_pred_ensemble = self.ensemble.predict(X_val)\n accuracy_ensemble = accuracy_score(y_val, y_pred_ensemble)\n \n try:\n if hasattr(self.ensemble, 'predict_proba') and len(np.unique(y_val)) > 1:\n y_prob_ensemble = self.ensemble.predict_proba(X_val)[:, 1]\n roc_auc_ensemble = roc_auc_score(y_val, y_prob_ensemble)\n else:\n roc_auc_ensemble = 0.5\n except:\n roc_auc_ensemble = 0.5\n \n results['ensemble'] = {\n 'accuracy': accuracy_ensemble,\n 'roc_auc': roc_auc_ensemble,\n 'classification_report': classification_report(y_val, y_pred_ensemble)\n }\n \n except Exception as e:\n print(f\"Failed to evaluate ensemble: {str(e)}\")\n \n return results\n\n# Initialize ML pipeline\nml_pipeline = AdvancedMLPipeline()\n\n# Check class distribution in the full dataset first\nprint(\"Checking original class distribution...\")\nunique_classes, class_counts = np.unique(y_train, return_counts=True)\nprint(f\"Original classes: {dict(zip(unique_classes, class_counts))}\")\nprint(f\"Class balance ratio: {class_counts.min()/class_counts.max():.3f}\")\n\n# Use a more conservative split to ensure both classes are represented\ntest_size = 0.3 if class_counts.min() > 10 else 0.15 # Smaller test size if few samples\n\n# Ensure minimum representation of minority class\nmin_class_samples = class_counts.min()\nif min_class_samples < 6:\n print(f\"Warning: Very few samples in minority class ({min_class_samples}). Consider getting more data.\")\n test_size = 0.1 # Very small test size\n\nprint(f\"Using test size: {test_size}\")\n\n# Split feature data with stratification\nX_train_feat, X_val_feat, y_train_feat, y_val_feat = train_test_split(\n X_train_features, y_train, test_size=test_size, random_state=42, stratify=y_train\n)\n\nprint(f\"Feature-based train shape: {X_train_feat.shape}\")\nprint(f\"Feature-based validation shape: {X_val_feat.shape}\")\n\n# Check class distribution after split\nprint(\"After train/validation split:\")\nprint(f\"Train classes: {dict(zip(*np.unique(y_train_feat, return_counts=True)))}\")\nprint(f\"Val classes: {dict(zip(*np.unique(y_val_feat, return_counts=True)))}\")\n\n# Prepare features\nX_train_processed, X_val_processed, selected_features = ml_pipeline.prepare_features(\n X_train_feat, y_train_feat, X_val_feat, feature_selection_k=200 # Reduced feature count\n)\n\nprint(f\"Processed train shape: {X_train_processed.shape}\")\nprint(f\"Processed validation shape: {X_val_processed.shape}\")\n\n# Train individual models\ncv_scores = ml_pipeline.train_individual_models(X_train_processed, y_train_feat)\n\n# Create ensemble\nml_pipeline.create_ensemble(X_train_processed, y_train_feat)\n\nprint(\"\\nTraditional ML models trained successfully!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-10T10:02:48.645804Z","iopub.execute_input":"2025-08-10T10:02:48.646483Z","iopub.status.idle":"2025-08-10T10:02:51.829962Z","shell.execute_reply.started":"2025-08-10T10:02:48.646458Z","shell.execute_reply":"2025-08-10T10:02:51.82904Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n \n \n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}