{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":87955,"databundleVersionId":10054796,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport numpy as np\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, TensorDataset, Dataset\nfrom sklearn.metrics import f1_score, balanced_accuracy_score\nfrom sklearn.preprocessing import StandardScaler\nimport os\nimport glob\nfrom sklearn.model_selection import StratifiedKFold\nfrom torch import nn\nimport pandas as pd\nfrom transformers import get_cosine_schedule_with_warmup\nfrom scipy.stats import mode","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:26:04.882250Z","iopub.execute_input":"2024-11-06T10:26:04.882578Z","iopub.status.idle":"2024-11-06T10:26:04.889937Z","shell.execute_reply.started":"2024-11-06T10:26:04.882543Z","shell.execute_reply":"2024-11-06T10:26:04.889037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeSeriesAugmentation:\n    def __init__(self, noise_level=0.05, shift_range=10):\n        self.noise_level = noise_level\n        self.shift_range = shift_range\n    \n    def add_noise(self, x):\n        noise = torch.randn_like(x) * self.noise_level\n        return x + noise\n    \n    def time_shift(self, x):\n        # Generate random shift value\n        shift = torch.randint(-self.shift_range, self.shift_range, (1,)).item()\n        # Roll the tensor along the time dimension\n        return torch.roll(x, shifts=shift, dims=0)\n    \n    def scale(self, x):\n        scale_factor = torch.rand(1) * 0.4 + 0.8  # Random scale between 0.8 and 1.2\n        return x * scale_factor\n\nclass AugmentedDataset(Dataset):\n    def __init__(self, X, y, augment=True):\n        self.X = X\n        self.y = y\n        self.augment = augment\n        self.aug = TimeSeriesAugmentation()\n    \n    def __getitem__(self, idx):\n        x = self.X[idx].clone()  # Create a copy to avoid modifying original data\n        if self.augment:\n            if torch.rand(1) < 0.5:\n                x = self.aug.add_noise(x)\n            if torch.rand(1) < 0.5:\n                x = self.aug.time_shift(x)\n            if torch.rand(1) < 0.5:\n                x = self.aug.scale(x)\n        return x, self.y[idx]\n    \n    def __len__(self):\n        return len(self.X)","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:15:10.225855Z","iopub.execute_input":"2024-11-06T10:15:10.226307Z","iopub.status.idle":"2024-11-06T10:15:10.239123Z","shell.execute_reply.started":"2024-11-06T10:15:10.226258Z","shell.execute_reply":"2024-11-06T10:15:10.238146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_dataloaders(batch_size=32, n_fold=0):\n    X_train = np.concatenate([np.load(f'/kaggle/input/bci-i-idun-eeg-analysis-challenge/S00{i}_X.npy') for i in range(1, 4)])\n    y_train = np.concatenate([np.load(f'/kaggle/input/bci-i-idun-eeg-analysis-challenge/S00{i}_y.npy') for i in range(1, 4)])\n    X_test = np.load('/kaggle/input/bci-i-idun-eeg-analysis-challenge/S004_X.npy')\n    X = np.concatenate([X_train, X_test])\n    mean_ = X.mean()\n    std_ = X.std()\n    X_train = (X_train - mean_) / std_\n    markers = sorted(list(set(y_train)))\n    y_train = [markers.index(x) for x in y_train]\n    \n    if n_fold == 0:\n        val_index = list(range(655))\n    elif n_fold == 1:\n        val_index = list(range(655, 1064 + 655))\n    elif n_fold == 2:\n        val_index = list(range(1064 + 655, 1064 + 655 + 823))\n    \n#     # Create train/validation split\n#     skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2024)\n#     for i, (train_index, val_index) in enumerate(skf.split(X_train, y_train)):\n#         if i == n_fold:\n#             break\n    train_index = [i for i in range(X_train.shape[0]) if i not in val_index]\n    \n    train_dataset = AugmentedDataset(torch.FloatTensor(X_train[train_index]), torch.LongTensor(y_train)[train_index])\n    val_dataset = AugmentedDataset(torch.FloatTensor(X_train[val_index]), torch.LongTensor(y_train)[val_index], False)\n    print(len(train_dataset))\n    print(len(val_dataset))\n\n    # Create DataLoaders\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n    \n    return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:20:45.308090Z","iopub.execute_input":"2024-11-06T10:20:45.308804Z","iopub.status.idle":"2024-11-06T10:20:45.321562Z","shell.execute_reply.started":"2024-11-06T10:20:45.308749Z","shell.execute_reply":"2024-11-06T10:20:45.320537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_one_epoch(model, train_loader, criterion, optimizer, scheduler, device):\n    model.train()\n    running_loss = 0.0\n    for images, labels in tqdm(train_loader):\n        images, labels = images.to(device), labels.to(device)\n\n        optimizer.zero_grad()\n        \n        outputs = model(images)\n        loss = criterion(outputs, labels)\n\n        # Backward pass and optimization step\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n\n        running_loss += loss.item() * images.size(0)\n\n    epoch_loss = running_loss / len(train_loader.dataset)\n    return epoch_loss\n\ndef validate(model, val_loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    all_labels = []\n    all_outputs = []\n    \n    with torch.no_grad():\n        for images, labels in tqdm(val_loader):\n            images, labels = images.to(device), labels.to(device)\n\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n\n            running_loss += loss.item() * images.size(0)\n            all_labels.append(labels.cpu().numpy())\n            all_outputs.append(outputs.cpu().numpy())\n\n    epoch_loss = running_loss / len(val_loader.dataset)\n    \n    all_labels = np.concatenate(all_labels)\n    all_outputs = np.concatenate(all_outputs)\n    all_outputs = torch.nn.functional.softmax(torch.tensor(all_outputs), dim=-1).numpy()  # Convert logits to probabilities\n    \n    return epoch_loss, all_labels, all_outputs","metadata":{"execution":{"iopub.status.busy":"2024-11-06T09:39:32.513481Z","iopub.execute_input":"2024-11-06T09:39:32.513969Z","iopub.status.idle":"2024-11-06T09:39:32.526058Z","shell.execute_reply.started":"2024-11-06T09:39:32.513925Z","shell.execute_reply":"2024-11-06T09:39:32.525153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=7, verbose=False, delta=0):\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n\n    def __call__(self, val_loss, model, path):\n        score = -val_loss\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model, path):\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), path + '/' + 'checkpoint.pth')\n        self.val_loss_min = val_loss","metadata":{"execution":{"iopub.status.busy":"2024-11-06T09:39:32.527203Z","iopub.execute_input":"2024-11-06T09:39:32.527950Z","iopub.status.idle":"2024-11-06T09:39:32.540097Z","shell.execute_reply.started":"2024-11-06T09:39:32.527908Z","shell.execute_reply":"2024-11-06T09:39:32.539249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_model(model, model_name, num_epochs=10, batch_size=32, lr=1e-4, n_fold=0, device='cuda', patience=3, warmup_epochs=0):\n    train_loader, val_loader = create_dataloaders(batch_size=batch_size, n_fold=n_fold)\n    train_sets = len(train_loader)\n\n    model     = model.to(device)\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=lr)\n    scheduler = get_cosine_schedule_with_warmup(optimizer, train_sets * warmup_epochs, train_sets * num_epochs)\n    early_stopping = EarlyStopping(patience=patience, verbose=True)\n\n    train_losses = []\n    val_losses   = []\n    \n    path = model_name + str(n_fold)\n    os.makedirs(path, exist_ok=True)\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        train_loss = train_one_epoch(model, train_loader, criterion, optimizer, scheduler, device)\n        val_loss, val_labels, val_outputs = validate(model, val_loader, criterion, device)\n\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        \n        # Calculate metrics on validation set\n        val_preds = val_outputs.argmax(axis=-1)\n        accuracy = balanced_accuracy_score(val_labels, val_preds)\n        f1 = f1_score(val_labels, val_preds, average='weighted')\n        print(f'Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}, Val Accuracy: {accuracy:.4f}, Val F1 : {f1:.4f}')\n        early_stopping(-f1, model, path)\n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break\n        print('Updating learning rate to {}'.format(scheduler.get_last_lr()[0]))\n        \n    # Plot Loss\n    plt.plot(train_losses, label='Train Loss')\n    plt.plot(val_losses, label='Validation Loss')\n    plt.legend()\n    plt.savefig('loss_plot.png')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-06T09:39:32.660198Z","iopub.execute_input":"2024-11-06T09:39:32.660479Z","iopub.status.idle":"2024-11-06T09:39:32.671929Z","shell.execute_reply.started":"2024-11-06T09:39:32.660450Z","shell.execute_reply":"2024-11-06T09:39:32.671179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TimeSeriesCNN(nn.Module):\n    def __init__(self, input_length, num_classes=7):\n        super(TimeSeriesCNN, self).__init__()\n        \n        # Convolutional layers\n        self.conv_layers = nn.Sequential(\n            nn.Conv1d(1, 32, kernel_size=8, stride=1, padding='same'),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.MaxPool1d(kernel_size=2, stride=2),\n            \n            nn.Conv1d(32, 64, kernel_size=5, stride=1, padding='same'),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.MaxPool1d(kernel_size=2, stride=2),\n            \n            nn.Conv1d(64, 128, kernel_size=3, stride=1, padding='same'),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.MaxPool1d(kernel_size=2, stride=2)\n        )\n        \n        # Calculate the size of flattened features\n        self.flatten_size = self._get_flatten_size(input_length)\n        \n        # Fully connected layers\n        self.fc_layers = nn.Sequential(\n            nn.Linear(self.flatten_size, 256),\n            nn.ReLU(),\n            nn.Dropout(0.5),\n            nn.Linear(256, num_classes)\n        )\n        \n    def _get_flatten_size(self, input_length):\n        # Helper function to calculate flattened size\n        x = torch.randn(1, 1, input_length)\n        x = self.conv_layers(x)\n        return x.shape[1] * x.shape[2]\n    \n    def forward(self, x):\n        # x shape: (batch_size, n_samples)\n        # Add channel dimension\n        x = x.unsqueeze(1)  # (batch_size, 1, n_samples)\n        \n        x = self.conv_layers(x)\n        x = x.view(x.size(0), -1)  # Flatten\n        x = self.fc_layers(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2024-11-06T09:41:44.299293Z","iopub.execute_input":"2024-11-06T09:41:44.300059Z","iopub.status.idle":"2024-11-06T09:41:44.311568Z","shell.execute_reply.started":"2024-11-06T09:41:44.300017Z","shell.execute_reply":"2024-11-06T09:41:44.310653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nbatch_size = 128\nfor i in range(3):\n    model = TimeSeriesCNN(7501)\n    train_model(model, 'TimeSeriesCNN', num_epochs=30, batch_size=batch_size, lr=1e-3, n_fold=i, device='cuda', patience=10, warmup_epochs=5)","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:30:58.976811Z","iopub.execute_input":"2024-11-06T10:30:58.977220Z","iopub.status.idle":"2024-11-06T10:33:15.572262Z","shell.execute_reply.started":"2024-11-06T10:30:58.977181Z","shell.execute_reply":"2024-11-06T10:33:15.571057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(model, model_name, batch_size=32, n_fold=0, device='cuda', hard=True):\n    X_train = np.concatenate([np.load(f'/kaggle/input/bci-i-idun-eeg-analysis-challenge/S00{i}_X.npy') for i in range(1, 4)])\n    y_train = np.concatenate([np.load(f'/kaggle/input/bci-i-idun-eeg-analysis-challenge/S00{i}_y.npy') for i in range(1, 4)])\n    X_test = np.load('/kaggle/input/bci-i-idun-eeg-analysis-challenge/S004_X.npy')\n    X = np.concatenate([X_train, X_test])\n    mean_ = X.mean()\n    std_ = X.std()\n    X_test = (X_test - mean_) / std_\n    markers = sorted(list(set(y_train)))\n    y_train = [markers.index(x) for x in y_train]\n    \n    model     = model.to(device) # load the model into the GPU\n    model.load_state_dict(torch.load(os.path.join(model_name + str(n_fold), 'checkpoint.pth')))\n    criterion = nn.CrossEntropyLoss()\n    \n    test_dataset = AugmentedDataset(torch.FloatTensor(X_test), torch.LongTensor(y_train)[:X_test.shape[0]], False)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n    _, _, outputs = validate(model, test_loader, criterion, device)\n    if not hard:\n        return outputs, markers\n    outputs = outputs.argmax(axis=-1)\n    return outputs, markers","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:36:29.164269Z","iopub.execute_input":"2024-11-06T10:36:29.164671Z","iopub.status.idle":"2024-11-06T10:36:29.175858Z","shell.execute_reply.started":"2024-11-06T10:36:29.164633Z","shell.execute_reply":"2024-11-06T10:36:29.174826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nsoft_preds = []\nfor i in range(3):\n    preds, markers = predict(model, 'TimeSeriesCNN', batch_size=batch_size, n_fold=i, device='cuda', hard=False)\n    soft_preds.append(preds)\nsoft_preds = np.stack(soft_preds, axis=0).mean(axis=0).argmax(axis=-1)\npreds = [markers[i] for i in soft_preds]\nIDs = np.arange(len(preds))\n\ndf = pd.DataFrame({\"ID\":IDs, \"Marker\": preds})\ndf.to_csv(\"soft_submission.csv\", index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:34:19.008082Z","iopub.execute_input":"2024-11-06T10:34:19.009039Z","iopub.status.idle":"2024-11-06T10:34:21.077554Z","shell.execute_reply.started":"2024-11-06T10:34:19.008996Z","shell.execute_reply":"2024-11-06T10:34:21.076330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nhard_preds = []\nfor i in range(3):\n    preds, markers = predict(model, 'TimeSeriesCNN', batch_size=batch_size, n_fold=i, device='cuda', hard=True)\n    hard_preds.append(np.array(preds))\nhard_preds = np.stack(hard_preds, axis=-1)\npreds = mode(hard_preds, axis=-1).mode\npreds = [markers[i] for i in preds]\nIDs = np.arange(len(preds))\n\ndf = pd.DataFrame({\"ID\":IDs, \"Marker\": preds})\ndf.to_csv(\"hard_submission.csv\", index=False)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-06T10:37:11.638142Z","iopub.execute_input":"2024-11-06T10:37:11.638526Z","iopub.status.idle":"2024-11-06T10:37:13.736069Z","shell.execute_reply.started":"2024-11-06T10:37:11.638489Z","shell.execute_reply":"2024-11-06T10:37:13.734829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}