{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Install and import dependencies","metadata":{}},{"cell_type":"code","source":"!pip install torch torchvision torchsummary","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:51:48.001240Z","iopub.execute_input":"2023-12-28T23:51:48.001671Z","iopub.status.idle":"2023-12-28T23:51:59.675005Z","shell.execute_reply.started":"2023-12-28T23:51:48.001640Z","shell.execute_reply":"2023-12-28T23:51:59.673950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport torch\nfrom torch import nn\nimport matplotlib.pyplot as plt\nimport os.path\nimport torchsummary\nfrom torch.utils.data import DataLoader, random_split","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:52:38.965079Z","iopub.execute_input":"2023-12-28T23:52:38.965470Z","iopub.status.idle":"2023-12-28T23:52:38.970870Z","shell.execute_reply.started":"2023-12-28T23:52:38.965438Z","shell.execute_reply":"2023-12-28T23:52:38.969934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inspect the training dataset","metadata":{}},{"cell_type":"code","source":"raw_training_dataset = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\nraw_training_dataset.head(10)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:51:59.685416Z","iopub.execute_input":"2023-12-28T23:51:59.685746Z","iopub.status.idle":"2023-12-28T23:52:02.257642Z","shell.execute_reply.started":"2023-12-28T23:51:59.685721Z","shell.execute_reply":"2023-12-28T23:52:02.256703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create a custom dataset and load a sample image","metadata":{}},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader\nfrom torchvision.transforms import ToTensor\n\nTOTAL_IMAGE_PIXELS = 784\n\nclass DigitDataset(Dataset):\n    def __init__(self, file_path, normalise_pixels=False):\n        self.df = pd.read_csv(file_path)\n        self.normalise_pixels = normalise_pixels\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        label = row.get(\"label\")\n        pixels = self.get_pixels_tensor(row)\n\n        # The submission dataset won't have a label column, so we return a placeholder label of -1\n        if label is None:\n            label = -1\n\n        return (pixels, label)\n\n    def get_pixels_tensor(self, row):\n        flattened_pixel_values = []\n        for pixel_index in range(TOTAL_IMAGE_PIXELS):\n            column_name = f\"pixel{pixel_index}\"\n            pixel_value = row[column_name]\n            flattened_pixel_values.append(pixel_value)\n\n        pixel_tensor = torch.tensor(flattened_pixel_values)\n        pixel_tensor = pixel_tensor.reshape(1, 28, 28)\n\n        if self.normalise_pixels:\n            pixel_tensor = pixel_tensor / 255\n\n        return pixel_tensor\n\nsample_training_dataset = DigitDataset(\"/kaggle/input/digit-recognizer/train.csv\", normalise_pixels=True)\n\n# Sample a row from the dataset\npixels, label = sample_training_dataset.__getitem__(100)\nplt.imshow(pixels.squeeze(), cmap=\"gray\")\nplt.title(f\"Digit: {label}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:52:41.351074Z","iopub.execute_input":"2023-12-28T23:52:41.351442Z","iopub.status.idle":"2023-12-28T23:52:44.015683Z","shell.execute_reply.started":"2023-12-28T23:52:41.351413Z","shell.execute_reply":"2023-12-28T23:52:44.014833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define the Neural Network","metadata":{}},{"cell_type":"code","source":"class Network(nn.Module):\n    def __init__(self, input_dim, output_dim, use_conv2d=True):\n        super().__init__()\n\n        if use_conv2d:\n            self.model = nn.Sequential(\n                nn.Conv2d(1, 32, kernel_size=(3, 3), padding=1, stride=1),\n                nn.ReLU(),\n                nn.MaxPool2d(kernel_size=(2, 2), stride=2),\n                nn.Flatten(start_dim=1),\n                nn.Linear(6272, 3136),\n                nn.ReLU(),\n                nn.Linear(3136, 784),\n                nn.ReLU(),\n                nn.Linear(784, output_dim),\n            )\n        else:\n            self.model = nn.Sequential(\n                nn.Flatten(start_dim=1),\n                nn.Linear(input_dim, 784),\n                nn.ReLU(),\n                nn.Linear(784, 784),\n                nn.ReLU(),\n                nn.Linear(784, output_dim),\n            )\n\n    def forward(self, x):\n        y = self.model(x)\n        return y","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:52:47.155482Z","iopub.execute_input":"2023-12-28T23:52:47.156408Z","iopub.status.idle":"2023-12-28T23:52:47.164921Z","shell.execute_reply.started":"2023-12-28T23:52:47.156374Z","shell.execute_reply":"2023-12-28T23:52:47.163876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define the training and eval loop","metadata":{}},{"cell_type":"code","source":"class Trainer():\n    def __init__(self, network, num_epochs, learning_rate, train_batch_limit=10, eval_batch_limit=2, eval_accuracy=None,\n                 device=\"cpu\", train_dataloader=None, eval_dataloader=None):\n        self.network = network\n        self.num_epochs = num_epochs\n        self.loss_fn = nn.CrossEntropyLoss()\n        self.optimiser = torch.optim.SGD(network.parameters(), lr=learning_rate, momentum=0.9)\n        self.train_batch_limit = train_batch_limit\n        self.eval_batch_limit = eval_batch_limit\n        self.eval_accuracy = eval_accuracy\n        self.device = device\n        self.train_dataloader = train_dataloader\n        self.eval_dataloader = eval_dataloader\n\n    def run_epoch_training(self, epoch):\n        print(f\"==== \\n[epoch #{epoch + 1} ]\\n====\")\n\n        total_loss = 0\n        num_accurate_predictions = 0\n        total_samples = 0\n        total_batches = 0\n\n        self.network.train()\n        for batch, (x, y) in enumerate(self.train_dataloader):\n            total_batches += 1\n            total_samples += len(x)\n\n            x = x.to(self.device)\n            y = y.to(self.device)\n\n            # Clear previous gradients\n            self.optimiser.zero_grad()\n\n            # Run inference\n            logits = self.network(x)\n            logits = logits.type(torch.float32)\n\n            # Compute loss, gradients and update the weights\n            loss = self.loss_fn(logits, y)\n            loss.backward()\n            self.optimiser.step()\n\n            total_loss += loss.item()\n\n            _, prediction = logits.max(dim=1)\n            num_accurate_predictions_in_batch = (prediction == y).type(torch.float32).sum().item()\n            num_accurate_predictions += num_accurate_predictions_in_batch\n\n            if batch == self.train_batch_limit:\n                break\n\n        accuracy = num_accurate_predictions / total_samples\n        total_loss /= total_batches\n        print(\n            f\"[Test] Accuracy %: {accuracy * 100}%. Loss: {total_loss}. Predicted {num_accurate_predictions} / {total_samples} items.\")\n\n        return accuracy, total_loss\n\n    def run_epoch_eval(self, epoch):\n        print(\"==Evals==\")\n\n        total_loss = 0\n        num_accurate_predictions = 0\n        total_samples = 0\n        total_batches = 0\n\n        self.network.eval()\n        with torch.no_grad():\n            for batch, (x, y) in enumerate(self.eval_dataloader):\n                total_batches += 1\n                total_samples += len(x)\n\n                x = x.to(self.device)\n                y = y.to(self.device)\n\n                logits = self.network(x)\n                logits = logits.type(torch.float32)\n\n                loss = self.loss_fn(logits, y)\n                total_loss += loss.item()\n\n                _, prediction = logits.max(dim=1)\n                num_accurate_predictions_in_batch = (prediction == y).type(torch.float32).sum().item()\n                num_accurate_predictions += num_accurate_predictions_in_batch\n\n                if batch == self.eval_batch_limit:\n                    break\n\n            accuracy = num_accurate_predictions / total_samples\n            total_loss /= total_batches\n            print(\n                f\"[Eval] Accuracy %: {accuracy * 100}%. Loss: {total_loss}. Predicted {num_accurate_predictions} / {total_samples} items.\")\n\n            return accuracy, total_loss\n\n    def train(self):\n        if self.train_dataloader is None or self.eval_dataloader is None:\n            raise Exception(\"missing dataloader for training or eval dataset\")\n\n        for epoch in range(self.num_epochs):\n            self.run_epoch_training(epoch)\n            eval_accuracy, _ = self.run_epoch_eval(epoch)\n\n            if self.eval_accuracy is not None and eval_accuracy >= self.eval_accuracy:\n                print(f\"🎉 Reached {self.eval_accuracy * 100}% eval accuracy. Ending training\")\n                break","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:52:53.556808Z","iopub.execute_input":"2023-12-28T23:52:53.557639Z","iopub.status.idle":"2023-12-28T23:52:53.576747Z","shell.execute_reply.started":"2023-12-28T23:52:53.557604Z","shell.execute_reply":"2023-12-28T23:52:53.575836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Build and train the network, so the submission inference can be ran ","metadata":{}},{"cell_type":"code","source":"def build_network(device):\n    # Split the dataset into training and eval datasets so we can run evals on unseen data\n    base_dataset = DigitDataset(\"/kaggle/input/digit-recognizer/train.csv\", normalise_pixels=True)\n    training_dataset, eval_dataset = random_split(base_dataset, [0.95, 0.05])\n\n    # Setup the dataloaders\n    train_dataloader = DataLoader(training_dataset, shuffle=True, batch_size=600, num_workers=2)\n    eval_dataloader = DataLoader(eval_dataset, shuffle=True, batch_size=150, num_workers=2)\n\n    # initialise the model and train it\n    network = Network(input_dim=28 * 28, output_dim=10, use_conv2d=True)\n    network.to(device)\n\n    print(torchsummary.summary(network, input_size=(1, 28, 28)))\n\n    trainer = Trainer(network,\n                      num_epochs=30,\n                      train_batch_limit=-1,\n                      eval_batch_limit=-1,\n                      learning_rate=0.1,\n                      eval_accuracy=0.98,\n                      device=device,\n                      train_dataloader=train_dataloader,\n                      eval_dataloader=eval_dataloader,\n                      )\n    trainer.train()\n\n    # Save the model's weights after training\n    torch.save(network.state_dict(), \"digit-recognizer.pth\")\n    print(\"✨ Successfully trained model and saved weights\")\n\n    return network\n\ndef run_submission_inference(device):\n    # Confirm that the model's weights are in the directory\n    if os.path.isfile(\"digit-recognizer.pth\") is False:\n        raise Exception(\"model checkpoint file not found\")\n\n    # Load the weights from the model\n    network = Network(input_dim=28 * 28, output_dim=10, use_conv2d=True)\n    network.to(device)\n\n    network.load_state_dict(torch.load(\"digit-recognizer.pth\"))\n    network.eval()\n\n    # Setup the test dataset and dataloader for submission\n    test_dataset = DigitDataset(\"/kaggle/input/digit-recognizer/test.csv\", normalise_pixels=True)\n    test_dataloader = DataLoader(test_dataset, batch_size=600, num_workers=2)\n\n    # Compute classifications for the test dataset\n    results = []\n    with torch.no_grad():\n        for batch, (x, y) in enumerate(test_dataloader):\n            x = x.to(device)\n\n            logits = network(x)\n            logits = logits.type(torch.float32)\n\n            _, prediction = logits.max(dim=1)\n            results = results + prediction.tolist()\n\n    # Save the computed results in a csv file\n    raw_submission_df = []\n    for index, label in enumerate(results):\n        # Create a row in the format (ImageId,Label) for each result\n        row = [index + 1, label]\n        raw_submission_df.append(row)\n\n    submission_df = pd.DataFrame(raw_submission_df, columns=[\"ImageId\", \"Label\"])\n    submission_df.to_csv(\"./submission.csv\", index=False)\n    print(\"✨ Successfully ran submission inference and generated submission CSV\")","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:54:08.009997Z","iopub.execute_input":"2023-12-28T23:54:08.010358Z","iopub.status.idle":"2023-12-28T23:54:08.024081Z","shell.execute_reply.started":"2023-12-28T23:54:08.010330Z","shell.execute_reply":"2023-12-28T23:54:08.023101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    _ = build_network(device)\n    run_submission_inference(device)","metadata":{"execution":{"iopub.status.busy":"2023-12-28T23:54:18.649911Z","iopub.execute_input":"2023-12-28T23:54:18.650248Z","iopub.status.idle":"2023-12-29T00:04:16.670388Z","shell.execute_reply.started":"2023-12-28T23:54:18.650224Z","shell.execute_reply":"2023-12-29T00:04:16.669321Z"},"trusted":true},"execution_count":null,"outputs":[]}]}