{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30822,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nimport random\nimport numpy as np\nimport pandas as pd\n\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nimport torchvision.transforms as transforms\n\nimport matplotlib.pyplot as plt\n\nfrom torch.utils.data import Subset, DataLoader\n\nfrom tqdm.auto import tqdm  # auto  choose the right version\nfrom torchvision.transforms.functional import rotate\n\n## YOUR CODE HERE\nnp.random.seed(42)\ntorch.manual_seed(42)\n\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(42)\n    torch.cuda.manual_seed_all(42)  # if using multiple GPUs\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntorch.set_default_device(device)\ntorch.set_default_dtype(torch.float32)\n\n# free gpu memory\ntorch.cuda.empty_cache()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:33.212599Z","iopub.execute_input":"2025-01-08T09:01:33.212906Z","iopub.status.idle":"2025-01-08T09:01:37.934724Z","shell.execute_reply.started":"2025-01-08T09:01:33.212877Z","shell.execute_reply":"2025-01-08T09:01:37.933918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\ntrue_labels = train_df['label'].values\ntrain_df.drop('label', axis=1, inplace=True)\n\n# normalize the data\ntrain_df = train_df / 255.0\ntest_df = test_df / 255.0","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:37.935747Z","iopub.execute_input":"2025-01-08T09:01:37.936056Z","iopub.status.idle":"2025-01-08T09:01:42.046587Z","shell.execute_reply.started":"2025-01-08T09:01:37.936036Z","shell.execute_reply":"2025-01-08T09:01:42.045557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MNISTNet(torch.nn.Module):\n    def __init__(self, n_hidden_neurons, degrees):\n        super().__init__()\n        self.degrees = degrees\n        self.fc1 = torch.nn.Linear(28 * 28, n_hidden_neurons)\n        self.act1 = torch.nn.LeakyReLU()\n        self.fc2 = torch.nn.Linear(n_hidden_neurons, 10)\n\n    def rotate_image_batch(self, batch):\n        device = batch.device\n        # Reshape for rotation\n        batch = batch.reshape(-1, 1, 28, 28)  # Add channel dimension\n        # Rotate using torch.nn.functional\n        batch = rotate(batch, self.degrees, \n                        interpolation=transforms.InterpolationMode.BILINEAR,\n                        fill=0\n                    )\n        return batch.reshape(-1, 28 * 28)\n\n    def forward(self, x):\n        # Keep data on GPU\n        if self. degrees != 0:\n            x = self.rotate_image_batch(x)\n        x = self.fc1(x)\n        x = self.act1(x)\n        res = self.fc2(x)\n        return res","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:42.048306Z","iopub.execute_input":"2025-01-08T09:01:42.048604Z","iopub.status.idle":"2025-01-08T09:01:42.054383Z","shell.execute_reply.started":"2025-01-08T09:01:42.048576Z","shell.execute_reply":"2025-01-08T09:01:42.053543Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train with data rotation 15 degrees\nbatch_size = 256\n\ntest_size = 0.1\n\n# X_train and X_test come from the train_df shuffled\n\nn_train = int(len(train_df) * (1 - test_size))\nindices = np.arange(len(train_df))\nnp.random.shuffle(indices)\n\ntrain_indices = indices[:n_train]\ntest_indices = indices[n_train:]\n\nX_train = train_df.iloc[train_indices].values\nX_test = train_df.iloc[test_indices].values\n\ny_train = true_labels[train_indices]\ny_test = true_labels[test_indices]\n\n# convert to tensor\n\nX_train = torch.tensor(X_train, dtype=torch.float32).to(device)\nX_test = torch.tensor(X_test, dtype=torch.float32).to(device)\n\ny_train = torch.tensor(y_train, dtype=torch.long).to(device)\ny_test = torch.tensor(y_test, dtype=torch.long).to(device)\n\ntrain_dataset = torch.utils.data.TensorDataset(X_train, y_train)\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset, \n    batch_size=batch_size, \n    shuffle=True,\n    pin_memory=False,\n    generator=torch.Generator(device=device)\n)\n\ntest_dataset = torch.utils.data.TensorDataset(X_test, y_test)\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, \n    batch_size=batch_size, \n    pin_memory=False, \n    shuffle=False\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:42.055521Z","iopub.execute_input":"2025-01-08T09:01:42.055736Z","iopub.status.idle":"2025-01-08T09:01:42.719838Z","shell.execute_reply.started":"2025-01-08T09:01:42.055718Z","shell.execute_reply":"2025-01-08T09:01:42.718909Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_hidden_neurons = 128\nmodels = []\ndegrees = [0, 15]\n\nfor degree in degrees:\n    model = MNISTNet(n_hidden_neurons, degree).to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-3)\n    loss_fn = torch.nn.CrossEntropyLoss()\n    models.append((model, optimizer, loss_fn))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:42.720694Z","iopub.execute_input":"2025-01-08T09:01:42.720931Z","iopub.status.idle":"2025-01-08T09:01:42.742324Z","shell.execute_reply.started":"2025-01-08T09:01:42.720902Z","shell.execute_reply":"2025-01-08T09:01:42.741514Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict(models: list, X_batch: torch.Tensor) -> torch.Tensor:\n    res = []\n    for model, _, _ in models:\n        model.eval()\n        with torch.no_grad():\n            y_pred = model(X_batch)\n            y_pred = F.softmax(y_pred, dim=1)\n            res.append(y_pred)\n    \n    res = torch.stack(res, dim=0)  \n    res = torch.sum(res, dim=0) \n    return res\n    \n\ndef train_batch(models, X_batch, y_batch):\n    for model, optimizer, loss_fn in models:\n        model.train()\n        optimizer.zero_grad()\n        y_pred = model(X_batch)\n        loss = loss_fn(y_pred, y_batch)\n        loss.backward()\n        optimizer.step()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:42.743250Z","iopub.execute_input":"2025-01-08T09:01:42.743490Z","iopub.status.idle":"2025-01-08T09:01:42.748450Z","shell.execute_reply.started":"2025-01-08T09:01:42.743471Z","shell.execute_reply":"2025-01-08T09:01:42.747774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_epochs = 100\n\ntrain_losses = []\ntrain_accuracies = [] \ntest_accuracies = []\n\nwith tqdm(total=n_epochs, desc='Training') as pbar:\n    for epoch in range(n_epochs):\n        # Training\n        model.train()\n        epoch_losses = []\n        for X_batch, y_batch in train_loader:\n            train_batch(models, X_batch, y_batch)\n            y_pred = predict(models, X_batch)\n            epoch_losses.append(loss_fn(y_pred, y_batch).item())\n            \n        train_losses.append(np.mean(epoch_losses))\n        \n        # Evaluation\n        model.eval()\n        with torch.no_grad():\n            # Train accuracy\n            correct, total = 0, 0\n            for X_batch, y_batch in train_loader:\n                y_pred = predict(models, X_batch)\n                correct += (y_pred.argmax(dim=1) == y_batch).sum().item()\n                total += len(y_batch)\n            train_acc = correct / total\n            train_accuracies.append(train_acc)\n            \n            # Test accuracy\n            correct, total = 0, 0\n            for X_batch, y_batch in test_loader:\n                X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n                y_pred = predict(models, X_batch)\n                correct += (y_pred.argmax(dim=1) == y_batch).sum().item()\n                total += len(y_batch)\n            test_acc = correct / total\n            test_accuracies.append(test_acc)\n\n        # Update progress bar\n        pbar.set_postfix({\n            'loss': f'{train_losses[-1]:.4f}',\n            'train_acc': f'{train_acc:.4f}',\n            'test_acc': f'{test_acc:.4f}'\n        })\n        pbar.update(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:01:42.749093Z","iopub.execute_input":"2025-01-08T09:01:42.749324Z","iopub.status.idle":"2025-01-08T09:05:16.632216Z","shell.execute_reply.started":"2025-01-08T09:01:42.749305Z","shell.execute_reply":"2025-01-08T09:05:16.631180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# plot the loss and training with test accuracy\n\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.title('Training Loss')\n\nplt.subplot(1, 2, 2)\n\nplt.plot(train_accuracies, label='Train')\nplt.plot(test_accuracies, label='Test')\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.title('Accuracy')\nplt.legend()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:05:16.633245Z","iopub.execute_input":"2025-01-08T09:05:16.633546Z","iopub.status.idle":"2025-01-08T09:05:17.037630Z","shell.execute_reply.started":"2025-01-08T09:05:16.633522Z","shell.execute_reply":"2025-01-08T09:05:17.036809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train model with all the data\n\nn_hidden_neurons = 128\nmodels = []\ndegrees = [0, 15]\n\nfor degree in degrees:\n    model = MNISTNet(n_hidden_neurons, degree).to(device)\n    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-3)\n    loss_fn = torch.nn.CrossEntropyLoss()\n    models.append((model, optimizer, loss_fn))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:07:23.374666Z","iopub.execute_input":"2025-01-08T09:07:23.374977Z","iopub.status.idle":"2025-01-08T09:07:23.382279Z","shell.execute_reply.started":"2025-01-08T09:07:23.374955Z","shell.execute_reply":"2025-01-08T09:07:23.381631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train with data rotation 15 degrees\nbatch_size = 256\n\nX_train = train_df.values\ny_train = true_labels\n\nX_train = torch.tensor(X_train, dtype=torch.float32).to(device)\ny_train = torch.tensor(y_train, dtype=torch.long).to(device)\n\ntrain_dataset = torch.utils.data.TensorDataset(X_train, y_train)\ntrain_loader = torch.utils.data.DataLoader(\n    train_dataset, \n    batch_size=batch_size, \n    shuffle=True,\n    pin_memory=False,\n    generator=torch.Generator(device=device)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:07:31.263828Z","iopub.execute_input":"2025-01-08T09:07:31.264118Z","iopub.status.idle":"2025-01-08T09:07:31.528818Z","shell.execute_reply.started":"2025-01-08T09:07:31.264093Z","shell.execute_reply":"2025-01-08T09:07:31.528158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_epochs = 100\n\nwith tqdm(total=n_epochs, desc='Training') as pbar:\n    for epoch in range(n_epochs):\n        # Training\n        model.train()\n        epoch_losses = []\n        for X_batch, y_batch in train_loader:\n            train_batch(models, X_batch, y_batch)\n            y_pred = predict(models, X_batch)\n            epoch_losses.append(loss_fn(y_pred, y_batch).item())\n            \n        train_losses.append(np.mean(epoch_losses))\n        \n        # Evaluation\n        model.eval()\n        with torch.no_grad():\n            # Train accuracy\n            correct, total = 0, 0\n            for X_batch, y_batch in train_loader:\n                y_pred = predict(models, X_batch)\n                correct += (y_pred.argmax(dim=1) == y_batch).sum().item()\n                total += len(y_batch)\n            train_acc = correct / total\n            train_accuracies.append(train_acc)\n\n        # Update progress bar\n        pbar.set_postfix({\n            'loss': f'{train_losses[-1]:.4f}',\n            'train_acc': f'{train_acc:.4f}',\n        })\n        pbar.update(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:08:08.422411Z","iopub.execute_input":"2025-01-08T09:08:08.422724Z","iopub.status.idle":"2025-01-08T09:11:58.277093Z","shell.execute_reply.started":"2025-01-08T09:08:08.422702Z","shell.execute_reply":"2025-01-08T09:11:58.275999Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Clear CUDA memory before starting\ntorch.cuda.empty_cache()\n\n# Set batch size\nbatch_size = 256\nall_predictions = []\n\nX_test = torch.tensor(test_df.values, dtype=torch.float32)\ntest_dataset = torch.utils.data.TensorDataset(X_test)\ntest_loader = torch.utils.data.DataLoader(\n    test_dataset, \n    batch_size=256,\n    shuffle=False,  \n    pin_memory=False,\n    generator=torch.Generator(device=device)\n)\n\n# Generate predictions\nall_predictions = []\nmodel = model.to(device)  # Ensure model is on correct device\nmodel.eval()\n\nwith torch.no_grad():\n    for X_batch in tqdm(test_loader, desc='Predicting'):\n        # Move batch to device\n        X_batch = X_batch[0].to(device)\n        y_pred = model(X_batch)\n        \n        # Move predictions back to CPU for numpy conversion\n        batch_preds = y_pred.argmax(dim=1).cpu().numpy()\n        all_predictions.extend(batch_preds)\n        \n        # Clean up if using GPU\n        if device.type == 'cuda':\n            del X_batch, y_pred, batch_preds\n            torch.cuda.empty_cache()\n\n# Create submission DataFrame\nsubmission_df = pd.DataFrame({\n    'ImageId': range(1, len(all_predictions) + 1),\n    'Label': all_predictions\n})\n\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-08T09:15:07.793448Z","iopub.execute_input":"2025-01-08T09:15:07.793762Z","iopub.status.idle":"2025-01-08T09:15:08.463572Z","shell.execute_reply.started":"2025-01-08T09:15:07.793733Z","shell.execute_reply":"2025-01-08T09:15:08.462852Z"}},"outputs":[],"execution_count":null}]}