{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MNIST Digit Recognition with higher accuracy using Elastic Distortion","metadata":{"papermill":{"duration":0.008616,"end_time":"2023-10-27T14:40:15.6548","exception":false,"start_time":"2023-10-27T14:40:15.646184","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Imports","metadata":{"papermill":{"duration":0.007913,"end_time":"2023-10-27T14:40:15.670948","exception":false,"start_time":"2023-10-27T14:40:15.663035","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import time\nimport os\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader, TensorDataset\n\nimport torchvision\nfrom torchvision import datasets, models, transforms\n\nfrom torch.utils.tensorboard import SummaryWriter\n\nfrom PIL import Image\nfrom tempfile import TemporaryDirectory\n\nwriter = SummaryWriter('/kaggle/working/logs')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"papermill":{"duration":12.841434,"end_time":"2023-10-27T14:40:28.520423","exception":false,"start_time":"2023-10-27T14:40:15.678989","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:37:49.175232Z","iopub.execute_input":"2023-11-06T16:37:49.175804Z","iopub.status.idle":"2023-11-06T16:38:03.756041Z","shell.execute_reply.started":"2023-11-06T16:37:49.175755Z","shell.execute_reply":"2023-11-06T16:38:03.754672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"PATH_TO_DATA = '/kaggle/input/digit-recognizer/'\ntrain_data = pd.read_csv(os.path.join(PATH_TO_DATA, 'train.csv'))\ntest_data = pd.read_csv(os.path.join(PATH_TO_DATA, 'test.csv'))\n\ntrain_data.info()","metadata":{"papermill":{"duration":5.670677,"end_time":"2023-10-27T14:40:34.199661","exception":false,"start_time":"2023-10-27T14:40:28.528984","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:03.757837Z","iopub.execute_input":"2023-11-06T16:38:03.758745Z","iopub.status.idle":"2023-11-06T16:38:10.795682Z","shell.execute_reply.started":"2023-11-06T16:38:03.758713Z","shell.execute_reply":"2023-11-06T16:38:10.794464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.head()","metadata":{"papermill":{"duration":0.030017,"end_time":"2023-10-27T14:40:34.238408","exception":false,"start_time":"2023-10-27T14:40:34.208391","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:10.797050Z","iopub.execute_input":"2023-11-06T16:38:10.797462Z","iopub.status.idle":"2023-11-06T16:38:10.803481Z","shell.execute_reply.started":"2023-11-06T16:38:10.797421Z","shell.execute_reply":"2023-11-06T16:38:10.802352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.shape, test_data.shape","metadata":{"papermill":{"duration":0.017195,"end_time":"2023-10-27T14:40:34.264792","exception":false,"start_time":"2023-10-27T14:40:34.247597","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:10.806203Z","iopub.execute_input":"2023-11-06T16:38:10.806901Z","iopub.status.idle":"2023-11-06T16:38:10.817683Z","shell.execute_reply.started":"2023-11-06T16:38:10.806843Z","shell.execute_reply":"2023-11-06T16:38:10.816421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing Dataset","metadata":{"papermill":{"duration":0.008641,"end_time":"2023-10-27T14:40:34.282426","exception":false,"start_time":"2023-10-27T14:40:34.273785","status":"completed"},"tags":[]}},{"cell_type":"code","source":"X = train_data.values[:, 1:]\ny = train_data.values[:, 0]\nX_test = test_data.values\nX.shape, y.shape","metadata":{"papermill":{"duration":0.017803,"end_time":"2023-10-27T14:40:34.309128","exception":false,"start_time":"2023-10-27T14:40:34.291325","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:10.819392Z","iopub.execute_input":"2023-11-06T16:38:10.820542Z","iopub.status.idle":"2023-11-06T16:38:10.831739Z","shell.execute_reply.started":"2023-11-06T16:38:10.820501Z","shell.execute_reply":"2023-11-06T16:38:10.830710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\nX_train.shape, X_val.shape, y_train.shape, y_val.shape","metadata":{"papermill":{"duration":0.583038,"end_time":"2023-10-27T14:40:34.901229","exception":false,"start_time":"2023-10-27T14:40:34.318191","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:10.833206Z","iopub.execute_input":"2023-11-06T16:38:10.834233Z","iopub.status.idle":"2023-11-06T16:38:11.024964Z","shell.execute_reply.started":"2023-11-06T16:38:10.834202Z","shell.execute_reply":"2023-11-06T16:38:11.023782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Dataset Classes","metadata":{}},{"cell_type":"code","source":"class MNISTDataset(Dataset):\n    def __init__(self, X, y, transform=None):\n        self.n_samples = X.shape[0]\n        X = X.reshape(-1, 1, 28, 28) / 255.\n        self.x_data = torch.from_numpy(X.astype(np.float32))\n        self.y_data = torch.from_numpy(y)\n        \n        self.transform = transform\n        \n    def __len__(self):\n        return self.n_samples\n    \n    def __getitem__(self, item):\n        image, label = self.x_data[item], self.y_data[item]\n        \n        if self.transform is not None:\n            image = self.transform(image)\n            \n        return image, label","metadata":{"collapsed":false,"papermill":{"duration":0.019264,"end_time":"2023-10-27T14:40:34.929963","exception":false,"start_time":"2023-10-27T14:40:34.910699","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-06T16:38:11.026257Z","iopub.execute_input":"2023-11-06T16:38:11.026584Z","iopub.status.idle":"2023-11-06T16:38:11.035574Z","shell.execute_reply.started":"2023-11-06T16:38:11.026555Z","shell.execute_reply":"2023-11-06T16:38:11.034195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MNISTDataset_test(Dataset):\n    def __init__(self, X, transform=None):\n        self.n_samples = X.shape[0]\n        X = X.reshape(-1, 1, 28, 28) / 255.\n        self.x_data = torch.from_numpy(X.astype(np.float32))\n        \n        self.transform = transform\n        \n    def __len__(self):\n        return self.n_samples\n    \n    def __getitem__(self, item):\n        sample = self.x_data[item]\n        \n        if self.transform is not None:\n            sample = self.transform(sample)\n            \n        return sample","metadata":{"collapsed":false,"papermill":{"duration":0.017957,"end_time":"2023-10-27T14:40:34.957047","exception":false,"start_time":"2023-10-27T14:40:34.93909","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-06T16:38:11.037075Z","iopub.execute_input":"2023-11-06T16:38:11.037665Z","iopub.status.idle":"2023-11-06T16:38:11.047569Z","shell.execute_reply.started":"2023-11-06T16:38:11.037634Z","shell.execute_reply":"2023-11-06T16:38:11.046611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Transformations","metadata":{"papermill":{"duration":0.009196,"end_time":"2023-10-27T14:40:34.975568","exception":false,"start_time":"2023-10-27T14:40:34.966372","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"Elastic distortion involves applying random displacements to the individual pixels or groups of pixels in an image. These displacements are often drawn from a Gaussian distribution, which means they are probabilistic and can vary in magnitude and direction. By applying such elastic deformations, the model learns to be more robust to variations in input images, which can help improve its generalization and performance on unseen data.\n\nUsage of this method for MNIST digit classification has been described in [\"Best Practices for Convolutional Neural Networks Applied to Visual Document Analysis\"](https://www.microsoft.com/en-us/research/wp-content/uploads/2003/08/icdar03.pdf)\n\nThe method outline:\n1. Create random displacement fields for height and width, with values randomly sampled from $\\mathrm{unif}(-1,1)$. A displacement field defines a direction and magnitude to move a pixel. \n2. Smooth the fields with a gaussian filter. Since $\\mu = 0$ for $\\mathrm{unif}(-1,1)$, most values will be close to $0$ after the gaussian filter is applied. Thus most of the changes made by the fields will be small (assuming the gaussian filter's sigma value is large enough).\n3. Multiply the fields by a scaling factor to control intensity of the deformations.\n4. Use interpolation to apply the displacement fields to the image.\n\n#### Elastic Distortion Function\n\nCredit to the following gists for the basic function:\n- https://gist.github.com/fmder/e28813c1e8721830ff9c\n- https://gist.github.com/chsasank/4d8f68caf01f041a6453e67fb30f8f5a\n- https://gist.github.com/erniejunior/601cdf56d2b424757de5\n\n[this notebook was used as reference for elastic transform part](https://www.kaggle.com/code/babbler/mnist-data-augmentation-with-elastic-distortion/notebook)","metadata":{}},{"cell_type":"code","source":"from scipy.ndimage.filters import gaussian_filter\nfrom scipy.ndimage.interpolation import map_coordinates\n\ndef elastic_transform(image, alpha_range, sigma, random_state=None):\n    \"\"\"Elastic deformation of images as described in [Simard2003]_.\n    .. [Simard2003] Simard, Steinkraus and Platt, \"Best Practices for\n       Convolutional Neural Networks applied to Visual Document Analysis\", in\n       Proc. of the International Conference on Document Analysis and\n       Recognition, 2003.\n       \n   # Arguments\n       image: Numpy array with shape (height, width, channels). \n       alpha_range: Float for fixed value or [lower, upper] for random value from uniform distribution.\n           Controls intensity of deformation.\n       sigma: Float, sigma of gaussian filter that smooths the displacement fields.\n       random_state: `numpy.random.RandomState` object for generating displacement fields.\n    \"\"\"\n    \n    if random_state is None:\n        random_state = np.random.RandomState(None)\n    \n    if np.isscalar(alpha_range):\n        alpha = alpha_range\n    else:\n        alpha = np.random.uniform(low=alpha_range[0], high=alpha_range[1])\n    \n    shape = image.shape\n    dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma) * alpha\n    dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma) * alpha\n    \n    x, y, z = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), np.arange(shape[2]), indexing='ij')\n    indices = np.reshape(x + dx, (-1, 1)), np.reshape(y + dy, (-1, 1)), np.reshape(z, (-1, 1))\n    \n    return torch.from_numpy(map_coordinates(image, indices, order=1, mode='reflect').reshape(1, 28, 28)).to(device, torch.float32)","metadata":{"collapsed":false,"papermill":{"duration":0.020965,"end_time":"2023-10-27T14:40:35.005826","exception":false,"start_time":"2023-10-27T14:40:34.984861","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.048999Z","iopub.execute_input":"2023-11-06T16:38:11.049326Z","iopub.status.idle":"2023-11-06T16:38:11.068917Z","shell.execute_reply.started":"2023-11-06T16:38:11.049299Z","shell.execute_reply":"2023-11-06T16:38:11.067745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = transforms.Compose([\n    transforms.ToPILImage(),\n    transforms.RandomRotation(15),  # Randomly rotate by up to 10 degrees\n    transforms.RandomAffine(degrees=0, translate=(0.15, 0.15), scale=(0.9, 1.1)),  # Randomly zoom and shift\n    transforms.Lambda(lambda x: elastic_transform(np.array(x).reshape(28, 28, 1), alpha_range=[0, 5], sigma=3)),\n#     transforms.ToTensor()\n])","metadata":{"collapsed":false,"papermill":{"duration":0.017024,"end_time":"2023-10-27T14:40:35.032162","exception":false,"start_time":"2023-10-27T14:40:35.015138","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.072887Z","iopub.execute_input":"2023-11-06T16:38:11.073277Z","iopub.status.idle":"2023-11-06T16:38:11.083509Z","shell.execute_reply.started":"2023-11-06T16:38:11.073248Z","shell.execute_reply":"2023-11-06T16:38:11.082543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating dataloaders on transformed datasets","metadata":{}},{"cell_type":"code","source":"batch_size = 128\nnum_classes = 10 ","metadata":{"collapsed":false,"papermill":{"duration":0.015521,"end_time":"2023-10-27T14:40:35.056674","exception":false,"start_time":"2023-10-27T14:40:35.041153","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.084580Z","iopub.execute_input":"2023-11-06T16:38:11.084941Z","iopub.status.idle":"2023-11-06T16:38:11.100454Z","shell.execute_reply.started":"2023-11-06T16:38:11.084911Z","shell.execute_reply":"2023-11-06T16:38:11.099230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = MNISTDataset(X_train, y_train, transform=transform_train)\nval_dataset = MNISTDataset(X_val, y_val)\n\nfull_train_dataset = MNISTDataset(X, y, transform=transform_train)\ntest_dataset = MNISTDataset_test(X_test)","metadata":{"collapsed":false,"papermill":{"duration":0.411019,"end_time":"2023-10-27T14:40:35.476855","exception":false,"start_time":"2023-10-27T14:40:35.065836","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.102034Z","iopub.execute_input":"2023-11-06T16:38:11.102428Z","iopub.status.idle":"2023-11-06T16:38:11.264004Z","shell.execute_reply.started":"2023-11-06T16:38:11.102398Z","shell.execute_reply":"2023-11-06T16:38:11.262879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\nval_loader = DataLoader(dataset=val_dataset, batch_size=batch_size, shuffle=True)\n\nfull_train_loader = DataLoader(dataset=full_train_dataset, batch_size=batch_size, shuffle=True)\ntest_loader = DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)\n\nsteps_per_epoch = len(train_loader)\nsteps_per_epoch","metadata":{"papermill":{"duration":0.019461,"end_time":"2023-10-27T14:40:35.506036","exception":false,"start_time":"2023-10-27T14:40:35.486575","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:11.265427Z","iopub.execute_input":"2023-11-06T16:38:11.266417Z","iopub.status.idle":"2023-11-06T16:38:11.276260Z","shell.execute_reply.started":"2023-11-06T16:38:11.266384Z","shell.execute_reply":"2023-11-06T16:38:11.274973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"images, labels = next(iter(train_loader))\nimages.shape, labels.shape","metadata":{"papermill":{"duration":5.219255,"end_time":"2023-10-27T14:40:40.734377","exception":false,"start_time":"2023-10-27T14:40:35.515122","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-11-06T16:38:11.277791Z","iopub.execute_input":"2023-11-06T16:38:11.278180Z","iopub.status.idle":"2023-11-06T16:38:11.602453Z","shell.execute_reply.started":"2023-11-06T16:38:11.278148Z","shell.execute_reply":"2023-11-06T16:38:11.601045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility Methods for training and eval","metadata":{"papermill":{"duration":0.00935,"end_time":"2023-10-27T14:40:40.753434","exception":false,"start_time":"2023-10-27T14:40:40.744084","status":"completed"},"tags":[],"_kg_hide-input":false}},{"cell_type":"code","source":"def train_model(model, lr, num_epochs, train_loader=train_loader, val_loader=None, scheduler=None):\n    \n    optimizer = optim.Adam(model.parameters(), lr=lr)\n    criterion = nn.CrossEntropyLoss()\n    \n    for epoch in range(num_epochs):\n\n        running_loss = 0.0\n        running_corrects = 0\n        running_corrects_val = 0\n            \n        for i, (images, labels) in enumerate(train_loader):\n            images = images.to(device)\n            labels = labels.to(device)            \n            \n            optimizer.zero_grad()\n            output = model(images)\n            loss = criterion(output, labels)\n            loss.backward()\n            optimizer.step()\n            \n            with torch.no_grad():\n                _, predictions = torch.max(output, 1)\n                corrects = (predictions == labels).sum().item()\n                running_corrects += corrects\n                \n                running_loss += loss.item()\n                \n            writer.add_scalar('train_loss', loss, epoch*steps_per_epoch + i)\n            writer.add_scalar('train_accuracy', corrects/labels.shape[0], epoch*steps_per_epoch + i)\n        \n        epoch_loss = running_loss / steps_per_epoch\n        epoch_acc = running_corrects*100/len(train_loader)/batch_size\n\n        print(f'epoch {epoch+1}/{num_epochs}, loss {epoch_loss:.4f}, training accuracy {epoch_acc:.2f}%', end='')\n                \n        if val_loader is not None:\n            for i, (images, labels) in enumerate(val_loader):\n                images = images.to(device)\n                labels = labels.to(device)\n                \n                with torch.no_grad():\n                    output = model(images)\n                    _, predictions = torch.max(output, 1)\n                    corrects = (predictions == labels).sum().item()\n                    running_corrects_val += corrects\n                    \n                writer.add_scalar('val_accuracy', corrects/labels.shape[0], epoch*steps_per_epoch + i)\n                    \n            epoch_val_acc = running_corrects_val*100/len(val_loader)/batch_size\n            print(f', validation accuracy {epoch_val_acc:.2f}%')\n        else:\n            print()\n                \n        if scheduler is not None:\n            scheduler.step()\n            \n    writer.flush()\n    writer.close()\n            ","metadata":{"collapsed":false,"papermill":{"duration":0.02452,"end_time":"2023-10-27T14:40:40.787351","exception":false,"start_time":"2023-10-27T14:40:40.762831","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-06T16:38:11.604176Z","iopub.execute_input":"2023-11-06T16:38:11.605414Z","iopub.status.idle":"2023-11-06T16:38:11.622194Z","shell.execute_reply.started":"2023-11-06T16:38:11.605369Z","shell.execute_reply":"2023-11-06T16:38:11.621119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix\n\ndef validate_model(model, val_loader=val_loader):\n    \n    y_true = []\n    y_pred = []\n    \n    with torch.no_grad():\n        for images, labels in val_loader:\n            images = images.to(device)\n            \n            output = model(images) \n            _, predictions = torch.max(output, 1)\n            \n            y_true.extend(labels.cpu().tolist())\n            y_pred.extend(predictions.cpu().tolist())\n            \n    print(classification_report(y_true, y_pred))\n    cm = confusion_matrix(y_true, y_pred)\n    sns.heatmap(cm, annot=True)","metadata":{"collapsed":false,"papermill":{"duration":0.019096,"end_time":"2023-10-27T14:40:40.816155","exception":false,"start_time":"2023-10-27T14:40:40.797059","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-06T16:38:11.625989Z","iopub.execute_input":"2023-11-06T16:38:11.626329Z","iopub.status.idle":"2023-11-06T16:38:11.637491Z","shell.execute_reply.started":"2023-11-06T16:38:11.626301Z","shell.execute_reply":"2023-11-06T16:38:11.636420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_model(model, test_loader=test_loader, y_true=None):\n    \n    y_pred = []\n    \n    with torch.no_grad():\n        for images in test_loader:\n            images = images.to(device)\n            \n            output = model(images) \n            _, predictions = torch.max(output, 1)\n            \n            y_pred.extend(predictions.cpu().tolist())\n        \n    if y_true is not None:\n        print(classification_report(y_true, y_pred))\n        cm = confusion_matrix(y_true, y_pred, normalize='true')\n        sns.heatmap(cm, annot=True, fmt='.2f')\n    else:\n        return y_pred ","metadata":{"collapsed":false,"papermill":{"duration":0.018072,"end_time":"2023-10-27T14:40:40.843674","exception":false,"start_time":"2023-10-27T14:40:40.825602","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-06T16:38:11.638848Z","iopub.execute_input":"2023-11-06T16:38:11.639611Z","iopub.status.idle":"2023-11-06T16:38:11.653470Z","shell.execute_reply.started":"2023-11-06T16:38:11.639579Z","shell.execute_reply":"2023-11-06T16:38:11.652457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Trying Different Model Architectures","metadata":{}},{"cell_type":"code","source":"class ConvNet(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.net = nn.Sequential(\n            nn.Conv2d(1, 8, 5, 1),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2),\n            nn.Conv2d(8, 16, 5, 1),\n            nn.ReLU(),\n            nn.MaxPool2d(2, 2),\n            nn.Flatten(),\n            nn.Linear(16 * 4 * 4, 10)\n        )\n\n    def forward(self, x):\n        return self.net(x)\n    \nclass ConvNet2(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv = nn.Sequential(\n            nn.Conv2d(1, 32, 3, 1),\n            nn.BatchNorm2d(32),\n            nn.ReLU(),\n            nn.Conv2d(32, 64, 3, 1),\n            nn.BatchNorm2d(64),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Conv2d(64, 128, 3, 1),\n            nn.BatchNorm2d(128),\n            nn.ReLU(),\n            nn.MaxPool2d(2)\n        )\n        self.fc = nn.Sequential(\n            nn.Linear(128*5*5, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Linear(256, 10)\n        )\n        \n    def forward(self, x):\n        x = self.conv(x)\n        x = x.flatten(start_dim=1)\n        x = self.fc(x)\n        return x","metadata":{"collapsed":false,"papermill":{"duration":0.02307,"end_time":"2023-10-27T14:40:40.875997","exception":false,"start_time":"2023-10-27T14:40:40.852927","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.655010Z","iopub.execute_input":"2023-11-06T16:38:11.655975Z","iopub.status.idle":"2023-11-06T16:38:11.669580Z","shell.execute_reply.started":"2023-11-06T16:38:11.655942Z","shell.execute_reply":"2023-11-06T16:38:11.668260Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training Model","metadata":{"papermill":{"duration":0.009275,"end_time":"2023-10-27T14:40:40.894505","exception":false,"start_time":"2023-10-27T14:40:40.88523","status":"completed"},"tags":[]}},{"cell_type":"code","source":"lr = 1e-4\nnum_epochs = 10\nmodel = ConvNet2().to(device)","metadata":{"collapsed":false,"papermill":{"duration":1051.954808,"end_time":"2023-10-27T14:58:12.858533","exception":false,"start_time":"2023-10-27T14:40:40.903725","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.671209Z","iopub.execute_input":"2023-11-06T16:38:11.671643Z","iopub.status.idle":"2023-11-06T16:38:11.705646Z","shell.execute_reply.started":"2023-11-06T16:38:11.671605Z","shell.execute_reply":"2023-11-06T16:38:11.704711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_model(model, lr=lr, num_epochs=num_epochs, val_loader=val_loader)","metadata":{"collapsed":false,"papermill":{"duration":1051.954808,"end_time":"2023-10-27T14:58:12.858533","exception":false,"start_time":"2023-10-27T14:40:40.903725","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:11.707012Z","iopub.execute_input":"2023-11-06T16:38:11.707769Z","iopub.status.idle":"2023-11-06T16:38:44.736764Z","shell.execute_reply.started":"2023-11-06T16:38:11.707734Z","shell.execute_reply":"2023-11-06T16:38:44.734503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate_model(model)","metadata":{"collapsed":false,"papermill":{"duration":0.47149,"end_time":"2023-10-27T14:58:13.340846","exception":false,"start_time":"2023-10-27T14:58:12.869356","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:44.738870Z","iopub.execute_input":"2023-11-06T16:38:44.739750Z","iopub.status.idle":"2023-11-06T16:38:45.589317Z","shell.execute_reply.started":"2023-11-06T16:38:44.739704Z","shell.execute_reply":"2023-11-06T16:38:45.588102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Final Training and Submission","metadata":{"papermill":{"duration":0.01092,"end_time":"2023-10-27T14:58:13.363105","exception":false,"start_time":"2023-10-27T14:58:13.352185","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def submit_prediction(y_pred, name='submission'):\n    submission = pd.DataFrame({'ImageId': range(1, len(y_pred)+1), 'Label': y_pred})\n    submission.to_csv(f'/kaggle/working/{name}.csv', index=False)","metadata":{"collapsed":false,"papermill":{"duration":0.02397,"end_time":"2023-10-27T15:30:49.372449","exception":false,"start_time":"2023-10-27T15:30:49.348479","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:45.590970Z","iopub.execute_input":"2023-11-06T16:38:45.591343Z","iopub.status.idle":"2023-11-06T16:38:45.597520Z","shell.execute_reply.started":"2023-11-06T16:38:45.591312Z","shell.execute_reply":"2023-11-06T16:38:45.596489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = test_model(model)\nsubmit_prediction(y_pred)","metadata":{"collapsed":false,"papermill":{"duration":0.406434,"end_time":"2023-10-27T15:30:49.792285","exception":false,"start_time":"2023-10-27T15:30:49.385851","status":"completed"},"tags":[],"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-06T16:38:45.599244Z","iopub.execute_input":"2023-11-06T16:38:45.599610Z","iopub.status.idle":"2023-11-06T16:39:05.323043Z","shell.execute_reply.started":"2023-11-06T16:38:45.599582Z","shell.execute_reply":"2023-11-06T16:39:05.321773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This gives 99% accuracy on Leaderboard.\n\nPlease upvote if you liked the notebook","metadata":{}}]}