{"metadata":{"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"papermill":{"default_parameters":{},"duration":110.462184,"end_time":"2024-01-02T13:00:04.988288","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2024-01-02T12:58:14.526104","version":"2.4.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom PIL import Image\nimport random\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader, Dataset\nfrom sklearn.model_selection import train_test_split\n\n%matplotlib inline","metadata":{"papermill":{"duration":7.673039,"end_time":"2024-01-02T12:58:26.654547","exception":false,"start_time":"2024-01-02T12:58:18.981508","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:13.635630Z","iopub.execute_input":"2024-01-06T17:49:13.636051Z","iopub.status.idle":"2024-01-06T17:49:20.186782Z","shell.execute_reply.started":"2024-01-06T17:49:13.636015Z","shell.execute_reply":"2024-01-06T17:49:20.185616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"papermill":{"duration":0.102874,"end_time":"2024-01-02T12:58:26.762700","exception":false,"start_time":"2024-01-02T12:58:26.659826","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:20.188450Z","iopub.execute_input":"2024-01-06T17:49:20.188950Z","iopub.status.idle":"2024-01-06T17:49:20.193868Z","shell.execute_reply.started":"2024-01-06T17:49:20.188916Z","shell.execute_reply":"2024-01-06T17:49:20.192701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntrain","metadata":{"papermill":{"duration":3.458219,"end_time":"2024-01-02T12:58:30.225819","exception":false,"start_time":"2024-01-02T12:58:26.767600","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:20.195180Z","iopub.execute_input":"2024-01-06T17:49:20.195610Z","iopub.status.idle":"2024-01-06T17:49:24.587369Z","shell.execute_reply.started":"2024-01-06T17:49:20.195578Z","shell.execute_reply":"2024-01-06T17:49:24.586143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest","metadata":{"papermill":{"duration":2.021455,"end_time":"2024-01-02T12:58:32.252834","exception":false,"start_time":"2024-01-02T12:58:30.231379","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:24.591111Z","iopub.execute_input":"2024-01-06T17:49:24.592820Z","iopub.status.idle":"2024-01-06T17:49:27.055823Z","shell.execute_reply.started":"2024-01-06T17:49:24.592766Z","shell.execute_reply":"2024-01-06T17:49:27.054371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')\nsample","metadata":{"papermill":{"duration":0.033543,"end_time":"2024-01-02T12:58:32.293196","exception":false,"start_time":"2024-01-02T12:58:32.259653","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:27.057638Z","iopub.execute_input":"2024-01-06T17:49:27.058036Z","iopub.status.idle":"2024-01-06T17:49:27.082663Z","shell.execute_reply.started":"2024-01-06T17:49:27.058008Z","shell.execute_reply":"2024-01-06T17:49:27.081220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop('label', axis = 1).to_numpy(dtype='float32')\ny = train['label'].values.astype('int64')\n\ntrain_X, validation_X, train_y, validation_y = train_test_split(X, y, test_size = 0.15, random_state = 32)\n\nassert X.shape[0] == y.shape[0], 'X and y must have the same number of rows'\n\nX.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:27.084319Z","iopub.execute_input":"2024-01-06T17:49:27.084800Z","iopub.status.idle":"2024-01-06T17:49:27.673507Z","shell.execute_reply.started":"2024-01-06T17:49:27.084760Z","shell.execute_reply":"2024-01-06T17:49:27.672524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.to_numpy(dtype = 'float32')\ntest.shape","metadata":{"papermill":{"duration":0.014532,"end_time":"2024-01-02T12:58:32.592889","exception":false,"start_time":"2024-01-02T12:58:32.578357","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:27.674788Z","iopub.execute_input":"2024-01-06T17:49:27.675119Z","iopub.status.idle":"2024-01-06T17:49:27.723837Z","shell.execute_reply.started":"2024-01-06T17:49:27.675091Z","shell.execute_reply":"2024-01-06T17:49:27.722762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 10, figsize=(12, 8))\n\nfor i in range(10):\n    digit = train_X[i].reshape(28, 28)\n    ax[i].imshow(digit, cmap=plt.cm.binary, interpolation='bilinear')\n    ax[i].set_axis_off()\n    ax[i].set_title(train_y[i])\n\nplt.show()","metadata":{"papermill":{"duration":0.572224,"end_time":"2024-01-02T12:58:33.171137","exception":false,"start_time":"2024-01-02T12:58:32.598913","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:27.725297Z","iopub.execute_input":"2024-01-06T17:49:27.725673Z","iopub.status.idle":"2024-01-06T17:49:28.357613Z","shell.execute_reply.started":"2024-01-06T17:49:27.725635Z","shell.execute_reply":"2024-01-06T17:49:28.356698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"no_class = len(set(y))\nno_class","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:28.358699Z","iopub.execute_input":"2024-01-06T17:49:28.359275Z","iopub.status.idle":"2024-01-06T17:49:28.375055Z","shell.execute_reply.started":"2024-01-06T17:49:28.359245Z","shell.execute_reply":"2024-01-06T17:49:28.373913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_transform = transforms.Compose([\n    transforms.Resize(28),\n    transforms.Grayscale(num_output_channels=1), \n    transforms.ToTensor(),\n    transforms.Normalize([0.5], [0.5]),\n    transforms.Lambda(lambda x: x.unsqueeze(0))    \n])","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:28.379521Z","iopub.execute_input":"2024-01-06T17:49:28.379949Z","iopub.status.idle":"2024-01-06T17:49:28.387728Z","shell.execute_reply.started":"2024-01-06T17:49:28.379916Z","shell.execute_reply":"2024-01-06T17:49:28.386407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ImageDataset(Dataset):\n\n    def __init__(self, images, labels=None, transform=None):\n        self.images = images\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.images)\n\n    def __getitem__(self, idx):\n        image = self.images[idx] \n        image = torch.from_numpy(image).float()\n        image = image.view(1, 28, 28) \n    \n        if self.labels is not None:  \n            label = torch.tensor(self.labels[idx])\n            return image, label\n        return image","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:28.389782Z","iopub.execute_input":"2024-01-06T17:49:28.390193Z","iopub.status.idle":"2024-01-06T17:49:28.401356Z","shell.execute_reply.started":"2024-01-06T17:49:28.390162Z","shell.execute_reply":"2024-01-06T17:49:28.400004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = ImageDataset(train_X, train_y, X_transform)\ntrain_dataloader = DataLoader(train_set, batch_size=32, shuffle=True)\n\nvalidation_set = ImageDataset(validation_X, validation_y, X_transform)\nvalidation_dataloader = DataLoader(validation_set, batch_size=32, shuffle=True)\n\nprediction_set = ImageDataset(test, None, X_transform)\nprediction_dataloader = DataLoader(prediction_set, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:28.402921Z","iopub.execute_input":"2024-01-06T17:49:28.403477Z","iopub.status.idle":"2024-01-06T17:49:28.418223Z","shell.execute_reply.started":"2024-01-06T17:49:28.403444Z","shell.execute_reply":"2024-01-06T17:49:28.417063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_images , train_labels = next(iter(train_dataloader))\n\ntrain_images.shape","metadata":{"papermill":{"duration":0.014681,"end_time":"2024-01-02T12:58:33.346469","exception":false,"start_time":"2024-01-02T12:58:33.331788","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:28.419920Z","iopub.execute_input":"2024-01-06T17:49:28.420395Z","iopub.status.idle":"2024-01-06T17:49:28.527328Z","shell.execute_reply.started":"2024-01-06T17:49:28.420333Z","shell.execute_reply":"2024-01-06T17:49:28.526125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_images = next(iter(prediction_dataloader))\n\npredicted_images.shape","metadata":{"execution":{"iopub.status.busy":"2024-01-06T17:49:28.528608Z","iopub.execute_input":"2024-01-06T17:49:28.529407Z","iopub.status.idle":"2024-01-06T17:49:28.537063Z","shell.execute_reply.started":"2024-01-06T17:49:28.529364Z","shell.execute_reply":"2024-01-06T17:49:28.536008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define the Neural Network model for digit recognition\nclass DigitRecognizerNN(nn.Module):\n    def __init__(self):\n        super(DigitRecognizerNN, self).__init__()\n        \n        # Convolutional layers\n        self.conv1 = nn.Conv2d(1, 16, kernel_size=3, padding=1)  \n        self.bn1 = nn.BatchNorm2d(16)\n        self.conv2 = nn.Conv2d(16, 64, kernel_size=3, padding=1)  \n        self.bn2 = nn.BatchNorm2d(64)\n        self.conv3 = nn.Conv2d(64, 128, kernel_size=3, padding=1)  \n        self.bn3 = nn.BatchNorm2d(128)\n        \n        \n        # Max pooling layer\n        self.pool = nn.MaxPool2d(2, 2, padding=1)\n\n        # Dropout layers\n        self.dropout1 = nn.Dropout(0.5)\n        self.dropout2 = nn.Dropout(0.3)\n        self.dropout3 = nn.Dropout(0.6)\n\n        # Fully connected layers\n        self.fc1 = nn.Linear(128 * 5 * 5, 512)  # Input size after max pooling\n        self.fc2 = nn.Linear(512, 64)  \n        self.fc3 = nn.Linear(64, 10)  # Output layer for 10 classes\n\n    def forward(self, x):\n        x = self.pool(F.silu(self.bn1(self.conv1(x))))  # Convolution, BatchNorm, silu, Pooling\n        x = self.pool(F.silu(self.bn2(self.conv2(x))))\n        x = self.pool(F.silu(self.bn3(self.conv3(x))))\n        # print(x.shape)\n        x = x.view(x.size(0), -1)  # Flatten the tensor for fully connected layers\n        \n        # Apply dropout\n        x = self.dropout1(x)\n        \n        # Fully connected layers with silu activations\n        x = F.silu(self.fc1(x))\n        x = self.dropout2(x)\n        x = F.silu(self.fc2(x))\n        x = self.dropout3(x)\n        # Output layer without activation (to be handled by loss function)\n        x = F.softmax(self.fc3(x))\n        return x\n\n# Instantiate the model and move it to the device (CPU or GPU)\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")  # Choose device based on availability\nmodel = DigitRecognizerNN().to(device)  # Move the model to the chosen device\n","metadata":{"papermill":{"duration":4.55259,"end_time":"2024-01-02T12:58:37.905694","exception":false,"start_time":"2024-01-02T12:58:33.353104","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:28.539205Z","iopub.execute_input":"2024-01-06T17:49:28.540128Z","iopub.status.idle":"2024-01-06T17:49:28.586948Z","shell.execute_reply.started":"2024-01-06T17:49:28.540090Z","shell.execute_reply":"2024-01-06T17:49:28.585513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loss Function\nloss_fn = nn.CrossEntropyLoss()\n# Optimizer\noptimizer = optim.Adam(model.parameters(), lr=0.005, weight_decay=0.001)\n# Learning Rate Scheduler\nscheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=8, gamma=0.1)","metadata":{"papermill":{"duration":0.014643,"end_time":"2024-01-02T12:58:37.928132","exception":false,"start_time":"2024-01-02T12:58:37.913489","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:28.588763Z","iopub.execute_input":"2024-01-06T17:49:28.589241Z","iopub.status.idle":"2024-01-06T17:49:28.596814Z","shell.execute_reply.started":"2024-01-06T17:49:28.589199Z","shell.execute_reply":"2024-01-06T17:49:28.595388Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(dataloader, model, loss_fn, optimizer, device):\n    # Get the total size of the dataset\n    size = len(dataloader.dataset)\n    # Set the model in training mode\n    model.train()\n    # Initialize variables to track loss and accuracy\n    running_loss = 0.0\n    correct = 0\n    total = 0\n\n    # Create a progress bar using tqdm to visualize training progress\n    with tqdm(total=len(dataloader), desc=\"Training\", unit=\"batch\") as pbar:\n        # Iterate through each batch in the dataloader\n        for batch, (X, y) in enumerate(dataloader):\n            # Move data to the specified device (GPU or CPU)\n            X, y = X.to(device), y.to(device)\n\n            # Zero the gradients to prepare for backpropagation\n            optimizer.zero_grad()\n            # Forward pass: compute predictions\n            pred = model(X)\n            # Calculate the loss\n            loss = loss_fn(pred, y)\n            # Backpropagation: compute gradients\n            loss.backward()\n            # Update model parameters using the optimizer\n            optimizer.step()\n\n            # Update running loss\n            running_loss += loss.item()\n            # Calculate predicted labels by finding the index of the maximum value along dim=1\n            predicted = torch.argmax(pred, dim=1)\n            # Update total number of samples processed\n            total += y.size(0)\n            # Update number of correctly predicted samples\n            correct += (predicted == y).sum().item()\n\n            # Update the progress bar\n            pbar.update(1)\n            # Update displayed metrics in the progress bar\n            pbar.set_postfix({'loss': running_loss / (batch + 1), 'accuracy': 100 * correct / total})\n\n    # Calculate epoch-level loss and accuracy\n    epoch_loss = running_loss / len(dataloader)\n    epoch_acc = 100 * correct / size\n    # Print epoch-level metrics\n    print(f\"End of Epoch: Avg loss: {epoch_loss:.8f}, Accuracy: {epoch_acc:.8f}%\\n\")\n\ndef validate(dataloader, model, loss_fn, device):\n    size = len(dataloader.dataset)\n    model.eval()\n    val_loss = 0.0\n    correct = 0\n    total = 0\n\n    with torch.no_grad():\n        with tqdm(total=len(dataloader), desc=\"Validation\", unit=\"batch\") as pbar:\n            for batch, (X, y) in enumerate(dataloader):\n                X, y = X.to(device), y.to(device)\n                pred = model(X)\n                loss = loss_fn(pred, y)\n\n                val_loss += loss.item()\n                predicted = torch.argmax(pred, dim=1)\n                total += y.size(0)\n                correct += (predicted == y).sum().item()\n\n                pbar.update(1)\n                pbar.set_postfix({'val_loss': val_loss / (batch + 1), 'val_accuracy': 100 * correct / total})\n\n    epoch_loss = val_loss / len(dataloader)\n    epoch_acc = 100 * correct / size\n    print(f\"End of Validation: Avg loss: {epoch_loss:.8f}, Accuracy: {epoch_acc:.8f}%\\n\")\n","metadata":{"papermill":{"duration":0.018641,"end_time":"2024-01-02T12:58:37.953349","exception":false,"start_time":"2024-01-02T12:58:37.934708","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:28.598924Z","iopub.execute_input":"2024-01-06T17:49:28.599415Z","iopub.status.idle":"2024-01-06T17:49:28.619619Z","shell.execute_reply.started":"2024-01-06T17:49:28.599373Z","shell.execute_reply":"2024-01-06T17:49:28.618231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop\nepochs = 10\n\n# Loop through each epoch\nfor epoch in range(epochs):\n    # Print the current epoch number\n    print(f\"Epoch {epoch + 1}\")\n    \n    # Execute the training process for the current epoch\n    train(train_dataloader, model, loss_fn, optimizer, device)\n    \n    # Update the learning rate using the scheduler after completing an epoch\n    scheduler.step()\n    \n# Validation after training\nvalidate(validation_dataloader, model, loss_fn, device)\n\n# Once all epochs are completed, print the end of training message\nprint(\"End of training. Saving model...\")\n\n# Save the trained model's state dictionary to a file named \"model.pth\"\ntorch.save(model.state_dict(), \"model.pth\")\n","metadata":{"papermill":{"duration":77.247317,"end_time":"2024-01-02T12:59:55.207206","exception":false,"start_time":"2024-01-02T12:58:37.959889","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:49:28.621042Z","iopub.execute_input":"2024-01-06T17:49:28.621514Z","iopub.status.idle":"2024-01-06T17:58:05.727919Z","shell.execute_reply.started":"2024-01-06T17:49:28.621480Z","shell.execute_reply":"2024-01-06T17:58:05.726905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validate(validation_dataloader, model, loss_fn, device)\n\n# Set the model to evaluation mode\nmodel.eval()\n\n# List to store predicted labels\npredictions = []\ntrue_labels = []\n\n# Disable gradient calculation since we're only predicting\nwith torch.no_grad():\n    # Iterate through the test data loader\n    for data in prediction_dataloader:\n        # Move input data to the same device as the model\n        data = data.to(device)\n\n        # Get predictions from the model\n        outputs = model(data)\n        # Extract the predicted labels by finding the index of the maximum value\n        _, predicted = torch.max(outputs.data, 1)\n        # Extend the predictions list with the predicted labels (move to CPU if necessary)\n        predictions.extend(predicted.cpu().tolist())\n        \n        # Extend true labels list (move to CPU if necessary)\n        true_labels.extend(data.cpu().tolist())\n\n# Create a DataFrame for submission with ImageId and predicted labels\nsubmission = pd.DataFrame({\n    \"ImageId\": range(1, len(predictions) + 1),  # ImageId starts from 1\n    \"Label\": predictions  # Predicted labels\n})\n\n# Save the submission DataFrame to a CSV file named 'digits.csv' without including the index\nsubmission.to_csv('digits_nn.csv', index=False)\n\n# Get a random selection of 20 indices from the dataset\nrandom_indices = random.sample(range(len(prediction_dataloader.dataset)), 20)\n\n# Define the number of rows and columns for the subplot grid\nnum_rows = 10\nnum_cols = 10\n\n# Create a figure and set its size\nplt.figure(figsize=(20, 20))\n\n# Loop through indices of images (replace this with your data)\nfor idx in range(num_rows * num_cols):\n    plt.subplot(num_rows, num_cols, idx + 1)\n    plt.imshow(prediction_dataloader.dataset[idx][0].squeeze(), cmap='gray')  # Display the image\n    plt.axis('off')  # Hide axes\n    plt.title(f\"Predicted: {predictions[idx]}\")  # Show the predicted label as title\n\nplt.tight_layout()  # Adjust layout\nplt.show()  # Show the plot","metadata":{"papermill":{"duration":2.607907,"end_time":"2024-01-02T12:59:58.940879","exception":false,"start_time":"2024-01-02T12:59:56.332972","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2024-01-06T17:58:05.729889Z","iopub.execute_input":"2024-01-06T17:58:05.730259Z","iopub.status.idle":"2024-01-06T17:58:32.046775Z","shell.execute_reply.started":"2024-01-06T17:58:05.730222Z","shell.execute_reply":"2024-01-06T17:58:32.045548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":1.193338,"end_time":"2024-01-02T13:00:01.215901","exception":false,"start_time":"2024-01-02T13:00:00.022563","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}