{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":102335,"databundleVersionId":12518947,"sourceType":"competition"}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:20.870352Z","iopub.execute_input":"2025-08-13T07:10:20.871112Z","iopub.status.idle":"2025-08-13T07:10:21.186024Z","shell.execute_reply.started":"2025-08-13T07:10:20.871074Z","shell.execute_reply":"2025-08-13T07:10:21.185402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --------------------------------------------\n# Full Transformer-based Model Pipeline for CMI\n# --------------------------------------------\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport pickle\nfrom sklearn.preprocessing import LabelEncoder\nfrom torch.utils.data import Dataset, DataLoader\nimport torch\nimport torch.nn as nn\nfrom kaggle_evaluation.cmi_inference_server import CMIInferenceServer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:21.187051Z","iopub.execute_input":"2025-08-13T07:10:21.187353Z","iopub.status.idle":"2025-08-13T07:10:23.478533Z","shell.execute_reply.started":"2025-08-13T07:10:21.187335Z","shell.execute_reply":"2025-08-13T07:10:23.477962Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Configuration\n# -------------------------------\nSEQUENCE_LENGTH = 300\nBATCH_SIZE = 32\nEPOCHS = 40\nLEARNING_RATE = 1e-4\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:23.479284Z","iopub.execute_input":"2025-08-13T07:10:23.479580Z","iopub.status.idle":"2025-08-13T07:10:23.534249Z","shell.execute_reply.started":"2025-08-13T07:10:23.479562Z","shell.execute_reply":"2025-08-13T07:10:23.533464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Features to Use\n# -------------------------------\nfeatures = [\n    'acc_x', 'acc_y', 'acc_z',\n    'rot_w', 'rot_x', 'rot_y', 'rot_z',\n    'thm_1', 'thm_2', 'thm_3', 'thm_4', 'thm_5',\n] + [f\"tof_{i}_v{j}\" for i in range(1, 6) for j in range(64)]  # Optional: comment out ToF for faster training","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:23.535871Z","iopub.execute_input":"2025-08-13T07:10:23.536148Z","iopub.status.idle":"2025-08-13T07:10:23.550244Z","shell.execute_reply.started":"2025-08-13T07:10:23.536130Z","shell.execute_reply":"2025-08-13T07:10:23.549599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Load and Prepare Train Data\n# -------------------------------\ntrain = pd.read_csv(\"/kaggle/input/cmi-detect-behavior-with-sensor-data/train.csv\")\nALL_GESTURES = sorted(train[\"gesture\"].dropna().unique())\nle = LabelEncoder()\nle.fit(ALL_GESTURES)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:23.550880Z","iopub.execute_input":"2025-08-13T07:10:23.551161Z","iopub.status.idle":"2025-08-13T07:10:45.303486Z","shell.execute_reply.started":"2025-08-13T07:10:23.551143Z","shell.execute_reply":"2025-08-13T07:10:45.302771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save label encoder\nwith open(\"label_encoder.pkl\", \"wb\") as f:\n    pickle.dump(le, f)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:45.304220Z","iopub.execute_input":"2025-08-13T07:10:45.304423Z","iopub.status.idle":"2025-08-13T07:10:45.308678Z","shell.execute_reply.started":"2025-08-13T07:10:45.304391Z","shell.execute_reply":"2025-08-13T07:10:45.308103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Dataset Class\n# -------------------------------\nclass SequenceDataset(Dataset):\n    def __init__(self, df, label_encoder):\n        self.sequences = []\n        self.labels = []\n        for sequence_id, group in df.groupby(\"sequence_id\"):\n            X = group[features].fillna(0).to_numpy(dtype=np.float32)\n            if X.shape[0] < SEQUENCE_LENGTH:\n                pad_len = SEQUENCE_LENGTH - X.shape[0]\n                X = np.pad(X, ((pad_len, 0), (0, 0)), mode=\"constant\")\n            else:\n                X = X[-SEQUENCE_LENGTH:]\n            self.sequences.append(X)\n            self.labels.append(group[\"gesture\"].iloc[0])\n        self.labels = label_encoder.transform(self.labels)\n\n    def __len__(self):\n        return len(self.sequences)\n\n    def __getitem__(self, idx):\n        return torch.tensor(self.sequences[idx]), torch.tensor(self.labels[idx])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:45.309700Z","iopub.execute_input":"2025-08-13T07:10:45.310000Z","iopub.status.idle":"2025-08-13T07:10:45.324501Z","shell.execute_reply.started":"2025-08-13T07:10:45.309976Z","shell.execute_reply":"2025-08-13T07:10:45.323821Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Transformer Model Definition\n# -------------------------------\nimport torch\nimport torch.nn as nn\nimport math\n\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_len=5000):\n        super().__init__()\n        position = torch.arange(0, max_len).unsqueeze(1)\n        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))\n        pe = torch.zeros(1, max_len, d_model)\n        pe[0, :, 0::2] = torch.sin(position * div_term)\n        pe[0, :, 1::2] = torch.cos(position * div_term)\n        self.register_buffer('pe', pe)\n\n    def forward(self, x):\n        return x + self.pe[:, :x.size(1)]\n\n\nclass AttentionPooling(nn.Module):\n    def __init__(self, d_model):\n        super().__init__()\n        self.attn = nn.Linear(d_model, 1)\n\n    def forward(self, x):\n        weights = torch.softmax(self.attn(x), dim=1)  # [B, T, 1]\n        return (x * weights).sum(dim=1)  # Weighted sum\n\n\nclass TransformerClassifier(nn.Module):\n    def __init__(self, input_dim, num_classes, num_heads=4, num_layers=2, dropout=0.1):\n        super().__init__()\n        self.embedding = nn.Linear(input_dim, 128)\n        self.pos_encoder = PositionalEncoding(128)  # ✅ Added here\n        encoder_layer = nn.TransformerEncoderLayer(\n            d_model=128, nhead=num_heads, batch_first=True, dropout=dropout\n        )\n        self.transformer = nn.TransformerEncoder(encoder_layer, num_layers=num_layers)\n        self.attn_pool = AttentionPooling(128)  # ✅ Attention pooling\n        self.classifier = nn.Sequential(\n            nn.LayerNorm(128),\n            nn.Linear(128, 64),\n            nn.ReLU(),\n            nn.Dropout(dropout),\n            nn.Linear(64, num_classes)\n        )\n\n    def forward(self, x):\n        x = self.embedding(x)\n        x = self.pos_encoder(x)  # ✅ Now it exists\n        x = self.transformer(x)\n        x = self.attn_pool(x)    # ✅ Smarter pooling\n        return self.classifier(x)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:45.325200Z","iopub.execute_input":"2025-08-13T07:10:45.325490Z","iopub.status.idle":"2025-08-13T07:10:45.340589Z","shell.execute_reply.started":"2025-08-13T07:10:45.325460Z","shell.execute_reply":"2025-08-13T07:10:45.339885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# -------------------------------\n# Training\n# -------------------------------\nfrom torch.amp import autocast, GradScaler  # For mixed precision\n\nscaler = GradScaler()  # Helps stability in mixed precision\n\ntrain_dataset = SequenceDataset(train, le)\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n\nmodel = TransformerClassifier(\n    input_dim=len(features), \n    num_classes=len(le.classes_)\n).to(device)\n\noptimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)  # AdamW > Adam for transformers\ncriterion = nn.CrossEntropyLoss()\n\nfor epoch in range(EPOCHS):\n    model.train()\n    total_loss = 0\n\n    for X_batch, y_batch in train_loader:\n        X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n        \n        optimizer.zero_grad(set_to_none=True)  # Slight speedup\n        \n        # Mixed precision training\n        with autocast(device_type=device.type, dtype=torch.float16):\n            outputs = model(X_batch)\n            loss = criterion(outputs, y_batch)\n        \n        scaler.scale(loss).backward()\n        \n        # Clip gradients for stability\n        scaler.unscale_(optimizer)\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        \n        scaler.step(optimizer)\n        scaler.update()\n\n        total_loss += loss.item()\n\n    print(f\"Epoch {epoch+1}/{EPOCHS} - Loss: {total_loss/len(train_loader):.4f}\")\n\n# Save final model\ntorch.save(model.state_dict(), \"transformer_model.pt\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:10:45.341380Z","iopub.execute_input":"2025-08-13T07:10:45.342054Z","iopub.status.idle":"2025-08-13T07:18:56.015397Z","shell.execute_reply.started":"2025-08-13T07:10:45.342032Z","shell.execute_reply":"2025-08-13T07:18:56.014670Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Prediction function\ndef predict(sequence: pd.DataFrame, demographics: pd.DataFrame) -> str:\n    if not isinstance(sequence, pd.DataFrame):\n        sequence = sequence.to_pandas()\n\n    # Select features and fill missing\n    for col in features:\n        if col not in sequence.columns:\n            sequence[col] = 0\n    X = sequence[features].fillna(0).to_numpy(dtype=np.float32)\n\n    # Pad or truncate\n    if len(X) < SEQUENCE_LENGTH:\n        pad_len = SEQUENCE_LENGTH - len(X)\n        X = np.pad(X, ((pad_len, 0), (0, 0)), mode=\"constant\")\n    else:\n        X = X[-SEQUENCE_LENGTH:]\n\n    # Predict\n    X_tensor = torch.tensor(X, dtype=torch.float32).unsqueeze(0).to(device)\n    with torch.no_grad():\n        output = model(X_tensor)\n        pred_idx = output.argmax(dim=1).item()\n        pred_label = le.inverse_transform([pred_idx])[0]\n\n    return str(pred_label)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:18:56.017307Z","iopub.execute_input":"2025-08-13T07:18:56.017675Z","iopub.status.idle":"2025-08-13T07:18:56.023769Z","shell.execute_reply.started":"2025-08-13T07:18:56.017656Z","shell.execute_reply":"2025-08-13T07:18:56.023054Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference Server\ninference_server = CMIInferenceServer(predict)\n\nif os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\"):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(\n            \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test.csv\",\n            \"/kaggle/input/cmi-detect-behavior-with-sensor-data/test_demographics.csv\"\n        )\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-08-13T07:18:56.024606Z","iopub.execute_input":"2025-08-13T07:18:56.024856Z","iopub.status.idle":"2025-08-13T07:18:57.608859Z","shell.execute_reply.started":"2025-08-13T07:18:56.024829Z","shell.execute_reply":"2025-08-13T07:18:57.608221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}