{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CNN based Digit Recognizer using PyTorch-lightning\n* This notebook is designed for those with intermediate-level knowledge of computer vision and machine learning.\n* To provide hands-on experience, we are using `lightning` to help go over the entire training and evaluation processes.\n* The result of this notebook is [0.9949, 0.9958]\n","metadata":{}},{"cell_type":"code","source":"!pip install lightning","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:56:55.321954Z","iopub.execute_input":"2024-09-23T14:56:55.322376Z","iopub.status.idle":"2024-09-23T14:57:10.745016Z","shell.execute_reply.started":"2024-09-23T14:56:55.322334Z","shell.execute_reply":"2024-09-23T14:57:10.744035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Import all libraries","metadata":{}},{"cell_type":"code","source":"# Import libraries\nimport glob\nimport random\nimport numpy as np\nimport pandas as pd \nfrom tqdm import tqdm\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import accuracy_score\n\n# Pytorch\nimport torch\nimport torch.nn as nn\nfrom torch.nn import init\nimport torch.optim as optim\nfrom torch.utils import data\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as transforms\nfrom torchvision.transforms import functional as TF\n\n# Lightning\nimport lightning as L\nimport pytorch_lightning as pl\nfrom lightning.pytorch import Trainer\nfrom lightning.pytorch.callbacks import RichProgressBar, ModelCheckpoint, LearningRateMonitor, EarlyStopping","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:10.747327Z","iopub.execute_input":"2024-09-23T14:57:10.748044Z","iopub.status.idle":"2024-09-23T14:57:17.434608Z","shell.execute_reply.started":"2024-09-23T14:57:10.747995Z","shell.execute_reply":"2024-09-23T14:57:17.433641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check the data","metadata":{}},{"cell_type":"code","source":"# Load data\n\ntrain_dataframe = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_dataframe = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:17.435721Z","iopub.execute_input":"2024-09-23T14:57:17.436150Z","iopub.status.idle":"2024-09-23T14:57:23.219162Z","shell.execute_reply.started":"2024-09-23T14:57:17.436116Z","shell.execute_reply":"2024-09-23T14:57:23.218358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the data size\nprint(f'Number of Trainset samples: {len(train_dataframe)}')\nprint(f'Number of Testset samples: {len(test_dataframe)}')","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:23.221122Z","iopub.execute_input":"2024-09-23T14:57:23.221433Z","iopub.status.idle":"2024-09-23T14:57:23.226266Z","shell.execute_reply.started":"2024-09-23T14:57:23.221400Z","shell.execute_reply":"2024-09-23T14:57:23.225387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataframe.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:23.227355Z","iopub.execute_input":"2024-09-23T14:57:23.227658Z","iopub.status.idle":"2024-09-23T14:57:23.256362Z","shell.execute_reply.started":"2024-09-23T14:57:23.227622Z","shell.execute_reply":"2024-09-23T14:57:23.255490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset & Dataloader\nIn this section, we will build a custom dataset and dataloader for MNIST dataset.","metadata":{}},{"cell_type":"markdown","source":"#### Custom MNIST dataset class","metadata":{}},{"cell_type":"code","source":"class MNIST(Dataset):\n    def __init__(self, set_name, augmentation):\n        self.set_name = set_name # It should be one betweeb 'train', and 'test'\n        self.augmentation = augmentation\n        if set_name == 'test':\n            # Load test data\n            self.data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv').values.astype('float32')\n            self.labels = None  # No labels in test set\n            print('Test data shape:', self.data.shape)\n        else:\n            # Load training data\n            df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\n            self.labels = df['label'].values.astype('int64')  # Ensure labels are integers\n            self.data = df.drop('label', axis=1).values.astype('float32')\n\n            if set_name == 'val':\n                # Select every 20th element\n                self.data = self.data[::20]\n                self.labels = self.labels[::20]\n                print('Validation data shape:', self.data.shape)\n            elif set_name == 'train':\n                # Remove every 20th element\n                indices = np.delete(np.arange(len(self.data)), np.arange(0, len(self.data), 20))\n                self.data = self.data[indices]\n                self.labels = self.labels[indices]\n                print('Training data shape:', self.data.shape)\n            else:\n                raise ValueError(\"set_name must be one of ['train', 'val', 'test']\")\n        \n        self.data = torch.from_numpy(self.data)\n        self.input_size = 28\n    \n    def __len__(self):\n        return len(self.data)\n    \n    # Member function for reshaping a flatten tensor 1 x 784 to 2D image tensor 28 x 28.\n    def flatten2img(self, row):\n        return row.reshape((28, 28))\n    \n    # This function is about how to get a data sample for a given index\n    def __getitem__(self, idx):\n        if self.set_name == 'test':\n            image = self.flatten2img(self.data[idx]) / 255.0\n            image = torch.unsqueeze(image, 0)\n            label = torch.zeros(10, dtype=torch.float16)\n            return image, label\n        image = self.flatten2img(self.data[idx]) / 255.0\n        image = torch.unsqueeze(image, 0)\n        label = torch.zeros(10, dtype=torch.float16)\n        label[int(self.labels[idx])] = 1.0\n        \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:40.181295Z","iopub.execute_input":"2024-09-23T14:57:40.182196Z","iopub.status.idle":"2024-09-23T14:57:40.199613Z","shell.execute_reply.started":"2024-09-23T14:57:40.182141Z","shell.execute_reply":"2024-09-23T14:57:40.198458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check the custom dataset","metadata":{}},{"cell_type":"code","source":"sampleset = MNIST('train', True)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:40.725509Z","iopub.execute_input":"2024-09-23T14:57:40.726486Z","iopub.status.idle":"2024-09-23T14:57:43.595144Z","shell.execute_reply.started":"2024-09-23T14:57:40.726434Z","shell.execute_reply":"2024-09-23T14:57:43.594144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = len(sampleset)\nrand_idx = random.randint(0, n)\nimg, label = sampleset[rand_idx]\nimg = img * 255.0\nplt.figure()\nplt.title(f'Sample: {torch.argmax(label).item()}')\nplt.imshow(img[0])","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:43.596797Z","iopub.execute_input":"2024-09-23T14:57:43.597101Z","iopub.status.idle":"2024-09-23T14:57:44.010044Z","shell.execute_reply.started":"2024-09-23T14:57:43.597069Z","shell.execute_reply":"2024-09-23T14:57:44.008865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Custom MNIST dataloader class","metadata":{}},{"cell_type":"code","source":"class MNIST_loader():\n    def __init__(self, set_name, augmentation, batch_size, n_workers, shuffle):\n        self.set_name = set_name\n        self.augmentation = augmentation\n        self.batch_size = batch_size\n        self.n_workers = n_workers\n        self.shuffle = shuffle\n\n    def get_data_loader(self):\n        dataset = MNIST(self.set_name, self.augmentation)\n        data_loader = data.DataLoader(\n            dataset,\n            batch_size=self.batch_size,\n            shuffle=self.shuffle,\n            num_workers=self.n_workers,\n            drop_last=False\n        )\n        return data_loader","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:57:44.011386Z","iopub.execute_input":"2024-09-23T14:57:44.011811Z","iopub.status.idle":"2024-09-23T14:57:44.019022Z","shell.execute_reply.started":"2024-09-23T14:57:44.011764Z","shell.execute_reply":"2024-09-23T14:57:44.017875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Main model: DigitRecognizer","metadata":{}},{"cell_type":"code","source":"class DigitRecognizer(L.LightningModule):\n    def __init__(self, n_class, learning_rate):\n        super().__init__()\n        self.n_class = n_class\n        self.learning_rate = learning_rate\n        \n        self.model = nn.Sequential(\n            nn.Conv2d(1, 16, kernel_size = 7, stride = 3, padding = 3),\n            nn.ReLU(),\n            nn.Conv2d(16, 64, kernel_size = 3, stride = 2),\n            nn.Flatten()\n        )\n        self.encoder_outdim = 1024\n        self.fc = nn.Sequential(\n            nn.Linear(self.encoder_outdim, self.n_class)\n        )\n        \n        self.cross_entropy = nn.CrossEntropyLoss()\n        self.save_hyperparameters()\n        \n        # Lists for plotting training / validation results\n        self.train_loss_list, self.val_loss_list, self.train_acc_list, self.val_acc_list = [], [], [], []\n        # Dictionary for submission\n        self.test_preds = {\n            'ImageId' : [],\n            'Label' : []\n        }\n    \n    def forward(self, imgs):\n        out = self.model(imgs)\n        logit_preds = self.fc(out)\n        return logit_preds\n    \n    def training_step(self, batch, batch_idx):\n        imgs, logit_targets = batch\n        logit_preds = self.forward(imgs)\n        preds, targets = torch.argmax(logit_preds, dim=-1), torch.argmax(logit_targets, dim=-1)\n        train_loss = self.cross_entropy(logit_preds, targets)\n        train_acc = (preds == targets).float().mean()\n        \n        training_log = {\n            'train_loss': train_loss,\n            'train_acc': train_acc,\n        }\n        self.train_loss_list.append(train_loss.item())\n        self.train_acc_list.append(train_acc.item())\n        self.log_dict(training_log, prog_bar=True)\n        \n        return train_loss\n    \n    def validation_step(self, batch, batch_idx):\n        imgs, logit_targets = batch\n        logit_preds = self.forward(imgs)\n        preds, targets = torch.argmax(logit_preds, dim=-1), torch.argmax(logit_targets, dim=-1)\n        val_loss = self.cross_entropy(logit_preds, targets)\n        val_acc = (preds == targets).float().mean()\n        \n        validation_log = {\n            'val_loss': val_loss,\n            'val_acc': val_acc,\n        }\n        self.val_loss_list.append(val_loss.item())\n        self.val_acc_list.append(val_acc.item())\n        self.log_dict(validation_log, prog_bar=True)\n    \n    def test_step(self, batch, batch_idx):\n        imgs, logit_targets = batch\n        logit_preds = self.forward(imgs)\n        logit_preds = logit_preds.cpu().detach()\n        label_pred = torch.argmax(logit_preds, dim=-1)[0]\n        self.test_preds['ImageId'].append(batch_idx+1)\n        self.test_preds['Label'].append(label_pred.item())\n        \n        \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate, weight_decay=1e-4)\n        return optimizer","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:58:10.715624Z","iopub.execute_input":"2024-09-23T14:58:10.716040Z","iopub.status.idle":"2024-09-23T14:58:10.733061Z","shell.execute_reply.started":"2024-09-23T14:58:10.716003Z","shell.execute_reply":"2024-09-23T14:58:10.731990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Check the DigitRecognizer with a random input","metadata":{}},{"cell_type":"code","source":"device = 'cuda:0' if torch.cuda.is_available() else 'cpu'\nrgb = torch.randn(64, 1, 28, 28).to(device)\nmodel = DigitRecognizer(10, 0.0001).to(device)\n\n# Get number of parameters in M\npytorch_total_params = sum(p.numel() for p in model.parameters())\nprint(f\"Number of parameters: {pytorch_total_params/1e6:.2f}M\")\n\ncls_preds = model(rgb)\n\nprint(cls_preds.shape)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:58:11.261381Z","iopub.execute_input":"2024-09-23T14:58:11.262002Z","iopub.status.idle":"2024-09-23T14:58:11.695715Z","shell.execute_reply.started":"2024-09-23T14:58:11.261959Z","shell.execute_reply":"2024-09-23T14:58:11.694711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training and Validation Pipelines","metadata":{}},{"cell_type":"code","source":"max_epochs = 10\nbatch_size = 32\nn_workers = 16\nn_class = 10\nlearning_rate = 1e-4\ndevice = 'cuda:0' if torch.cuda.is_available() else 'cpu'\n\n# Apply the datasets on our custom data loader\ntrain_loader = MNIST_loader('train', True, batch_size=batch_size, n_workers=n_workers, shuffle=True).get_data_loader()\nval_loader = MNIST_loader('val', False, batch_size=batch_size, n_workers=n_workers, shuffle=False).get_data_loader()\n\n\n# Load the main model\nmodel = DigitRecognizer(n_class, learning_rate)\n\n# Define checkpoint callback\ncheckpoint_callback = ModelCheckpoint(\n    monitor='val_loss',\n    dirpath='/kaggle/working',\n    filename=f'DR-{{epoch:04d}}-{{val_loss:.6f}}',\n    save_top_k=1,\n    mode='min',\n)\n\ntrainer = Trainer(\n    min_epochs=1,\n    max_epochs=max_epochs,\n    precision=\"bf16\",\n    accelerator=\"gpu\",\n    devices=1,\n    callbacks=[checkpoint_callback]\n)\ntrainer.fit(model, train_loader, val_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T14:58:12.754537Z","iopub.execute_input":"2024-09-23T14:58:12.755383Z","iopub.status.idle":"2024-09-23T15:01:21.098321Z","shell.execute_reply.started":"2024-09-23T14:58:12.755342Z","shell.execute_reply":"2024-09-23T15:01:21.097420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot the results","metadata":{}},{"cell_type":"code","source":"train_losses, val_losses = model.train_loss_list, model.val_loss_list\ntrain_accs, val_accs = model.train_acc_list, model.val_acc_list\nsteps = range(len(train_losses))\nplt.figure()\nplt.title('Graph of Train Loss')\nplt.plot(steps, train_losses, label='Train Loss')\nplt.legend()\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.show()\n\nplt.figure()\nplt.title('Graph of Train Acc')\nplt.plot(steps, train_accs, label='Train Acc')\nplt.legend()\nplt.xlabel('Steps')\nplt.ylabel('Acc')\nplt.show()\n\nsteps = range(len(val_losses))\nplt.figure()\nplt.title('Graph of Val Loss')\nplt.plot(steps, val_losses, label='Val Loss')\nplt.legend()\nplt.xlabel('Steps')\nplt.ylabel('Loss')\nplt.show()\n\nplt.figure()\nplt.title('Graph of Val Acc')\nplt.plot(steps, val_accs, label='Val Acc')\nplt.legend()\nplt.xlabel('Steps')\nplt.ylabel('Acc')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:01:21.100235Z","iopub.execute_input":"2024-09-23T15:01:21.100593Z","iopub.status.idle":"2024-09-23T15:01:22.093622Z","shell.execute_reply.started":"2024-09-23T15:01:21.100534Z","shell.execute_reply":"2024-09-23T15:01:22.092689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Run Testing","metadata":{}},{"cell_type":"code","source":"n_workers = 16\nn_class = 10\nlearning_rate = 1e-6\ntest_loader = MNIST_loader('test', False, batch_size=1, n_workers=n_workers, shuffle=False).get_data_loader()\n\ntrained_models = glob.glob('/kaggle/working/*.ckpt')\nbest_model = None\nmin_val = 1e6\nfor m in trained_models:\n    ckpt_name = m.rsplit('.', 1)[0]\n    val_loss = float(ckpt_name.split('/')[-1].split('-val_loss=')[-1])\n    if val_loss < min_val:\n        min_val = val_loss\n        best_model = m\nprint(f'The best model is {best_model}')\n    \n# Load models:\nmodel = DigitRecognizer.load_from_checkpoint(\n    checkpoint_path=best_model,\n    n_class=n_class,\n    learning_rate=learning_rate\n).cuda()\ntrainer = Trainer(\n    min_epochs=1,\n    max_epochs=1,\n    precision=\"bf16\",\n    accelerator=\"gpu\",\n    devices=1\n)\ntrainer.test(model, test_loader)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:01:22.094851Z","iopub.execute_input":"2024-09-23T15:01:22.095199Z","iopub.status.idle":"2024-09-23T15:03:17.415285Z","shell.execute_reply.started":"2024-09-23T15:01:22.095164Z","shell.execute_reply":"2024-09-23T15:03:17.414224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save results and submit","metadata":{}},{"cell_type":"code","source":"test_data = model.test_preds","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:03:17.417733Z","iopub.execute_input":"2024-09-23T15:03:17.418090Z","iopub.status.idle":"2024-09-23T15:03:17.422556Z","shell.execute_reply.started":"2024-09-23T15:03:17.418052Z","shell.execute_reply":"2024-09-23T15:03:17.421707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output = pd.DataFrame(test_data)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:03:17.423971Z","iopub.execute_input":"2024-09-23T15:03:17.424286Z","iopub.status.idle":"2024-09-23T15:03:17.458425Z","shell.execute_reply.started":"2024-09-23T15:03:17.424254Z","shell.execute_reply":"2024-09-23T15:03:17.457621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:03:17.459759Z","iopub.execute_input":"2024-09-23T15:03:17.460116Z","iopub.status.idle":"2024-09-23T15:03:17.470557Z","shell.execute_reply.started":"2024-09-23T15:03:17.460074Z","shell.execute_reply":"2024-09-23T15:03:17.469625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output.to_csv('submission.csv', index=False, sep=',')","metadata":{"execution":{"iopub.status.busy":"2024-09-23T15:03:17.471831Z","iopub.execute_input":"2024-09-23T15:03:17.472154Z","iopub.status.idle":"2024-09-23T15:03:17.514980Z","shell.execute_reply.started":"2024-09-23T15:03:17.472102Z","shell.execute_reply":"2024-09-23T15:03:17.514144Z"},"trusted":true},"execution_count":null,"outputs":[]}]}