{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Preparation","metadata":{}},{"cell_type":"markdown","source":"## Imports","metadata":{}},{"cell_type":"code","source":"# packages\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport sys\n\n# PyTorch\nimport torch\nimport torch.optim\nfrom torch import nn\nfrom torchvision import transforms\n\n# Dataset to train and test\nfrom torch.utils.data import DataLoader, Dataset\nfrom IPython.display import clear_output","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:35:48.300885Z","start_time":"2023-07-13T11:35:48.297920Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:36:55.018221Z","iopub.execute_input":"2023-07-28T11:36:55.018588Z","iopub.status.idle":"2023-07-28T11:36:58.342886Z","shell.execute_reply.started":"2023-07-28T11:36:55.018561Z","shell.execute_reply":"2023-07-28T11:36:58.341833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from IPython.core.ultratb import VerboseTB\nVerboseTB._tb_highlight = \"bg:#e68cc3\"\n\nif \"dracula\" in plt.style.available:\n    plt.style.use(\"dracula\")","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:02:42.682169Z","start_time":"2023-07-13T11:02:42.680939Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:36:58.348327Z","iopub.execute_input":"2023-07-28T11:36:58.351027Z","iopub.status.idle":"2023-07-28T11:36:58.358332Z","shell.execute_reply.started":"2023-07-28T11:36:58.350974Z","shell.execute_reply":"2023-07-28T11:36:58.356861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Devices\n\nGPU device depends on OS which is used. For my personal usage, I work either from Windows or from Mac.","metadata":{}},{"cell_type":"code","source":"cpu_device = torch.device(\"cpu\")\n\n# Windows with NVidia GPU\nif torch.cuda.is_available():\n    gpu_device = torch.device(\"cuda\")\n\n# Mac with Apple Silicon\nif torch.has_mps:\n    gpu_device = torch.device(\"mps\")","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:02:42.685229Z","start_time":"2023-07-13T11:02:42.684068Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:37:04.216908Z","iopub.execute_input":"2023-07-28T11:37:04.217333Z","iopub.status.idle":"2023-07-28T11:37:04.297224Z","shell.execute_reply.started":"2023-07-28T11:37:04.217302Z","shell.execute_reply":"2023-07-28T11:37:04.296024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparing Dataset","metadata":{}},{"cell_type":"markdown","source":"## Distribution of data\n\nAmount of samples for each number is balanced so there's no need to create new data / remove the existing one.","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\n\nsns.barplot(x=df[\"label\"].value_counts().keys().values, y=df[\"label\"].value_counts().values)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:02:43.512982Z","start_time":"2023-07-13T11:02:43.433460Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:39:36.046821Z","iopub.execute_input":"2023-07-28T11:39:36.047210Z","iopub.status.idle":"2023-07-28T11:39:39.217957Z","shell.execute_reply.started":"2023-07-28T11:39:36.047178Z","shell.execute_reply":"2023-07-28T11:39:39.216961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define transforms\n\ntorchvision has transforms module, which allows to make transformations to data packed in Dataset and Dataloader classes.\n\nTransforms will be applied to each sample of data when picked from torch.utils.data.Dataset.\n\nAt first pixel values are in range [0, 255]. We transform it to range [0, 1] and change its type to torch.float32.\n\nFor train data we apply Random Rotation: it will rotate image randomly a little to augment training data and create new samples.\n\nThen we normalize data.","metadata":{}},{"cell_type":"code","source":"ROTATION_ANGLE = 2\n\nlabel_transforms = transforms.Compose([\n    transforms.Lambda(lambda x: torch.tensor(x).to(torch.long)),\n])\ntest_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Lambda(lambda x: (x/255.0).to(torch.float32)),\n    transforms.Normalize((0.1307,), (0.3081,)),\n])\ntrain_transforms = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Lambda(lambda x: (x/255.0).to(torch.float32)),\n    transforms.RandomRotation(degrees=(-ROTATION_ANGLE, ROTATION_ANGLE)),\n    transforms.Normalize((0.1307,), (0.3081,)),\n])","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:02:43.515942Z","start_time":"2023-07-13T11:02:43.514418Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:39:44.054693Z","iopub.execute_input":"2023-07-28T11:39:44.055090Z","iopub.status.idle":"2023-07-28T11:39:44.063505Z","shell.execute_reply.started":"2023-07-28T11:39:44.055057Z","shell.execute_reply":"2023-07-28T11:39:44.062317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define custom Dataset\n\nA custom Dataset should have at least 3 magic methods:\n    - __init__(*args, **kwargs) - accepts transforms and other params;\n    - __len__() - returns size of dataset;\n    - __getitem__(idx) - return item with applied transformations to it;\n\nMnistDataset constructor accepts filename, which contains data for training and testing, transforms that will be applied to data, bool parameter, which tells whether dataset is for training or not.","metadata":{}},{"cell_type":"code","source":"class MnistDataset(Dataset):\n    def __init__(self, annotations_file: str, transform: transforms.Compose=None,\n                 target_transform: transforms.Compose=None, train: bool=False):\n        df = pd.read_csv(annotations_file)\n        self.img_pixels = df.loc[:, df.columns!=\"label\"].values.reshape(df.__len__(), 28, 28)\n        self.transform = transform\n        self.target_transform = target_transform\n        self.train = train\n        self.img_labels = df[\"label\"].astype(\"long\").values if self.train else None\n\n    def __len__(self):\n        return len(self.img_pixels)\n\n    def __getitem__(self, idx):\n        image = self.img_pixels[idx]\n        if self.transform:\n            image = self.transform(image)\n        if self.train:\n            label = self.img_labels[idx]\n            if self.target_transform:\n                label = self.target_transform(label)\n            return image, label\n        else:\n            return image","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:02:43.519095Z","start_time":"2023-07-13T11:02:43.517647Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:39:47.142889Z","iopub.execute_input":"2023-07-28T11:39:47.143294Z","iopub.status.idle":"2023-07-28T11:39:47.152877Z","shell.execute_reply.started":"2023-07-28T11:39:47.143265Z","shell.execute_reply":"2023-07-28T11:39:47.151837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Dataloaders\n\nAfter defining Dataset we can create Dataloader, which accepts our MnistDataset, batch_size, shuffle.\n    - batch_size - describes, how big will be batch of data, when iterating over dataloader; the bigger the size of batch, the more data will be loaded into RAM;\n    - shuffle - bool param, which tells, whether the data should be shuffled when iterating over dataloader;\n\nIt's worth shuffling train_loader for network robustness to new data;\n\n","metadata":{}},{"cell_type":"code","source":"mnist_train_dataset = MnistDataset(\"/kaggle/input/digit-recognizer/train.csv\", transform=train_transforms, target_transform=label_transforms, train=True)\nmnist_test_dataset = MnistDataset(\"/kaggle/input/digit-recognizer/test.csv\", transform=test_transforms, train=False)\n\nmnist_train_loader = DataLoader(mnist_train_dataset, batch_size=128, shuffle=False)\nmnist_test_loader = DataLoader(mnist_test_dataset, batch_size=128, shuffle=False)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:27:37.126563Z","start_time":"2023-07-13T11:27:35.668580Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:40:01.645879Z","iopub.execute_input":"2023-07-28T11:40:01.646859Z","iopub.status.idle":"2023-07-28T11:40:06.668155Z","shell.execute_reply.started":"2023-07-28T11:40:01.646825Z","shell.execute_reply":"2023-07-28T11:40:06.667130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Network\n\nMNIST dataset is a kind of \"Hello world!\" in CV. The best kind of network for such data is convolutional. Let's define Convolutional Neural Network (CNN):\n<ol>\n    <li>At first it has 2 Convolutional layers with ReLU activation function.</li>\n    <li>Maxpooling is applied to the output with kernel size == 2: from a square of 4 pixels the one with the highest value is picked.</li>\n    <li>Dropout is used to reduce overfitting.</li>\n    <li>Then the data is flattened and passed to Linear layers.</li>\n    <li>Finally LogSoftmax is applied to get probabilities for numbers.</li>\n</ol>\n\nSequential() - takes sequence of nn.Modules and passes data through them in the order the layers are written into it when defining.","metadata":{}},{"cell_type":"code","source":"class CNN(nn.Module):\n    def __init__(self):\n        super(CNN, self).__init__()\n        self.layers = nn.Sequential(\n            nn.Conv2d(1, 32, 3),\n            nn.ReLU(),\n            nn.Conv2d(32, 64, 3),\n            nn.ReLU(),\n            nn.MaxPool2d(2),\n            nn.Dropout(0.25),\n            nn.Flatten(),\n            nn.Linear(9216, 256),\n            nn.ReLU(),\n            nn.Linear(256, 10),\n            nn.LogSoftmax(dim=1)\n        )\n\n    def forward(self, input):\n        output = self.layers(input)\n        return output","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:42:00.495577Z","start_time":"2023-07-13T11:42:00.488388Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:40:20.392055Z","iopub.execute_input":"2023-07-28T11:40:20.392462Z","iopub.status.idle":"2023-07-28T11:40:20.400565Z","shell.execute_reply.started":"2023-07-28T11:40:20.392431Z","shell.execute_reply":"2023-07-28T11:40:20.399205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function for training\n\nNow we define train function, which accepts:\n<ul>\n    <li>Main parameters: network, loader, loss function, optimizer</li>\n    <li>Hyperparameters: learning rate, epoch number</li>\n    <li>Other parameters: whether to use GPU, to plot and to verbose</li>\n</ul>\n\nAfter creating necessary variables and switching network to train we begin.\n\nDuring one epoch we iterate over dataloader to get data and labels.\nIn one iteration we zero gradients, make prediction, calculate loss, do backprop, and make optimization step. Moreover we save accuracies and losses to plot it if we want.\n\nAfter that we write additional output and draw accuracies/losses plots if we want to examine process of learning.","metadata":{}},{"cell_type":"code","source":"def train(network: nn.Module, train_loader: DataLoader,\n          loss: nn.modules.loss = None, optim: torch.optim = None,\n          learning_rate: float = 0.01, epochs: int =10, gpu: bool = False,\n          plot: bool = False, verbose: bool = False):\n    loss = loss or nn.NLLLoss()\n    optimizer = optim or torch.optim.Adam(network.parameters(), lr=learning_rate)\n\n    train_losses = []\n    train_accuracies = []\n    train_preds = []\n\n    network.to(device=gpu_device) if gpu else network.to(device=cpu_device)\n    network.train()\n    for i in tqdm(range(epochs)):\n        accuracies = []\n        losses = []\n        preds = []\n        for batch_x, batch_y in train_loader:\n            if gpu:\n                batch_x = batch_x.to(device=gpu_device)\n                batch_y = batch_y.to(device=gpu_device)\n            optimizer.zero_grad()\n            pred = network(batch_x)\n            batch_loss = loss(pred, batch_y)\n            losses.append(batch_loss.item())\n            batch_loss.backward()\n            optimizer.step()\n\n            accuracies.append((np.argmax(pred.cpu().data.numpy(), axis=1) == batch_y.cpu().data.numpy()).mean())\n            preds.append(pred)\n        train_losses.append(np.mean(losses))\n        train_accuracies.append(np.mean(accuracies))\n        train_preds.append(preds)\n        if verbose:\n            clear_output(True)\n            sys.stdout.write(f\"Epoch {i}: Train loss/accuracy: {train_losses[i]:.3f}/{train_accuracies[i]:.3f}\")\n        if plot:\n            plt.figure(figsize=(12, 5))\n            plt.subplot(1, 2, 1)\n            plt.xlabel(\"Epoch\")\n            plt.ylabel(\"Losses\")\n            plt.plot(train_losses)\n            plt.grid(\"on\")\n            plt.subplot(1, 2, 2)\n            plt.xlabel(\"Epoch\")\n            plt.ylabel(\"Accuracy\")\n            plt.plot(train_accuracies)\n            plt.grid(\"on\")\n            plt.show()\n    return train_accuracies, train_losses, train_preds","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:42:04.074203Z","start_time":"2023-07-13T11:42:04.071212Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:40:23.233900Z","iopub.execute_input":"2023-07-28T11:40:23.234587Z","iopub.status.idle":"2023-07-28T11:40:23.249530Z","shell.execute_reply.started":"2023-07-28T11:40:23.234554Z","shell.execute_reply":"2023-07-28T11:40:23.248371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Evaluation\n\nNot it's time to create our network and train.","metadata":{}},{"cell_type":"code","source":"cnn = CNN()\ntr_ac, tr_ls, predictions = train(cnn, mnist_train_loader, learning_rate=0.001,\n                                  epochs=20, loss=nn.CrossEntropyLoss(), gpu=True,\n                                  verbose=True, plot=True)","metadata":{"collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:40:25.371555Z","iopub.execute_input":"2023-07-28T11:40:25.372054Z","iopub.status.idle":"2023-07-28T11:50:57.186865Z","shell.execute_reply.started":"2023-07-28T11:40:25.371996Z","shell.execute_reply":"2023-07-28T11:50:57.185835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see incorrect labels.","metadata":{}},{"cell_type":"code","source":"count = 1\nplt.suptitle(\"(ground truth) / prediction\")\nfor idx, (X, y) in enumerate(mnist_train_loader):\n    if count > 25:\n        break\n    for i in range(128):\n        if count <= 25 and y[i].item() != predictions[-1][idx][i].argmax().item():\n            plt.subplot(5,5,count)\n            plt.imshow(X[i][0], cmap=plt.cm.Greys_r)\n            plt.title(f\"({y[i].item()})/{predictions[-1][idx][i].argmax().item()}\")\n            plt.axis(\"off\")\n            plt.tight_layout()\n            count += 1","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:44:44.656149Z","start_time":"2023-07-13T11:44:35.980368Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:50:57.188920Z","iopub.execute_input":"2023-07-28T11:50:57.189728Z","iopub.status.idle":"2023-07-28T11:51:23.295902Z","shell.execute_reply.started":"2023-07-28T11:50:57.189692Z","shell.execute_reply":"2023-07-28T11:51:23.294988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test prediction\n\nNow let's make prediction for kaggle test dataset. We switch network mode to evaluation, disable gradient computation and make predictions.","metadata":{}},{"cell_type":"code","source":"def test(network: nn.Module, test_loader: DataLoader, gpu: bool = False):\n    network.to(device=gpu_device) if gpu else network.to(device=cpu_device)\n    network.eval()\n    predictions = []\n    with torch.no_grad():\n        for X in tqdm(test_loader):\n            if gpu:\n                X = X.to(device=gpu_device)\n            preds = network(X)\n            predictions.append(preds)\n\n    return predictions","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:46:32.170909Z","start_time":"2023-07-13T11:46:32.163376Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:56:02.997157Z","iopub.execute_input":"2023-07-28T11:56:02.997706Z","iopub.status.idle":"2023-07-28T11:56:03.010114Z","shell.execute_reply.started":"2023-07-28T11:56:02.997661Z","shell.execute_reply":"2023-07-28T11:56:03.008860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = test(cnn, mnist_test_loader, gpu=True)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:46:34.035421Z","start_time":"2023-07-13T11:46:33.202068Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:56:09.222457Z","iopub.execute_input":"2023-07-28T11:56:09.222831Z","iopub.status.idle":"2023-07-28T11:56:12.808421Z","shell.execute_reply.started":"2023-07-28T11:56:09.222793Z","shell.execute_reply":"2023-07-28T11:56:12.807381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finally we prepare dataset and save it into .csv file.","metadata":{}},{"cell_type":"code","source":"submission = []\ni = 1\nfor pred_batch in tqdm(predictions):\n    for probs in pred_batch:\n        submission.append([i, probs.argmax().item()])\n        i += 1\nresult = pd.DataFrame(submission, columns=[\"ImageID\", \"Label\"])\nresult.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"collapsed":false,"ExecuteTime":{"end_time":"2023-07-13T11:48:09.938454Z","start_time":"2023-07-13T11:48:04.498056Z"},"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-07-28T11:56:59.935630Z","iopub.execute_input":"2023-07-28T11:56:59.936038Z","iopub.status.idle":"2023-07-28T11:57:01.144532Z","shell.execute_reply.started":"2023-07-28T11:56:59.935980Z","shell.execute_reply":"2023-07-28T11:57:01.143478Z"},"trusted":true},"execution_count":null,"outputs":[]}]}