{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from typing import Tuple, Union\n\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport csv\nimport random\n\nimport torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms\nfrom torch.utils.data import DataLoader\nfrom torch.utils.data import SubsetRandomSampler\nimport torch.nn.functional as F\n\nfrom torch.utils.data import SubsetRandomSampler\n\nimport cv2\n\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport sklearn.metrics\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.model_selection import train_test_split\n\nfrom hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n\nfrom PIL import Image\n\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:12.782602Z","iopub.execute_input":"2023-04-01T18:57:12.783045Z","iopub.status.idle":"2023-04-01T18:57:17.238939Z","shell.execute_reply.started":"2023-04-01T18:57:12.783005Z","shell.execute_reply":"2023-04-01T18:57:17.237673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading in data.","metadata":{}},{"cell_type":"markdown","source":"Lets load in the data first. After we have loaded in the data lets check the data with *.head*","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:17.241051Z","iopub.execute_input":"2023-04-01T18:57:17.241752Z","iopub.status.idle":"2023-04-01T18:57:22.435560Z","shell.execute_reply.started":"2023-04-01T18:57:17.241720Z","shell.execute_reply":"2023-04-01T18:57:22.434543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.437215Z","iopub.execute_input":"2023-04-01T18:57:22.437611Z","iopub.status.idle":"2023-04-01T18:57:22.467422Z","shell.execute_reply.started":"2023-04-01T18:57:22.437571Z","shell.execute_reply":"2023-04-01T18:57:22.466305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Class distribution","metadata":{}},{"cell_type":"markdown","source":"When we loaded in the data we then wanna check how the label distribution is within the dataframe. It is important if we are dealing with unbalanced dataset so we can prepare our data for such occurence. To check how many values there are for each label we can simply count the values based on the label within our dataframe. We then sort the data based on the label on the x axis so its more readable","metadata":{}},{"cell_type":"code","source":"class_value_counts = pd.DataFrame({\"label\": train_df['label'].value_counts().index, \"count\": train_df['label'].value_counts()})","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.470387Z","iopub.execute_input":"2023-04-01T18:57:22.470793Z","iopub.status.idle":"2023-04-01T18:57:22.482136Z","shell.execute_reply.started":"2023-04-01T18:57:22.470753Z","shell.execute_reply":"2023-04-01T18:57:22.480922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class_value_counts = class_value_counts.sort_values(\"label\")","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.483523Z","iopub.execute_input":"2023-04-01T18:57:22.484160Z","iopub.status.idle":"2023-04-01T18:57:22.492952Z","shell.execute_reply.started":"2023-04-01T18:57:22.484021Z","shell.execute_reply":"2023-04-01T18:57:22.491878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(class_value_counts, x=\"label\", y=\"count\")\nplt.title(\"class distribution within train_df\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.494600Z","iopub.execute_input":"2023-04-01T18:57:22.495443Z","iopub.status.idle":"2023-04-01T18:57:22.738647Z","shell.execute_reply.started":"2023-04-01T18:57:22.495403Z","shell.execute_reply":"2023-04-01T18:57:22.737472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transforming categorical targets","metadata":{}},{"cell_type":"markdown","source":"Since these are categorical targets we can do a pd dummies operation on them so we can use them as targets for our model output. After we did the pd dummies operation we then concatenate the original dataframe and our created target dataframe. in the end we drop the label column since we dont need it for reference anymore.","metadata":{}},{"cell_type":"code","source":"y_labels_hot_encoded = pd.get_dummies(train_df.label, prefix='y')","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.740363Z","iopub.execute_input":"2023-04-01T18:57:22.741109Z","iopub.status.idle":"2023-04-01T18:57:22.749763Z","shell.execute_reply.started":"2023-04-01T18:57:22.741067Z","shell.execute_reply":"2023-04-01T18:57:22.748688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.concat([y_labels_hot_encoded, train_df], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.751703Z","iopub.execute_input":"2023-04-01T18:57:22.752120Z","iopub.status.idle":"2023-04-01T18:57:22.854226Z","shell.execute_reply.started":"2023-04-01T18:57:22.752051Z","shell.execute_reply":"2023-04-01T18:57:22.853159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.drop(['label'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.855810Z","iopub.execute_input":"2023-04-01T18:57:22.856278Z","iopub.status.idle":"2023-04-01T18:57:22.939424Z","shell.execute_reply.started":"2023-04-01T18:57:22.856237Z","shell.execute_reply":"2023-04-01T18:57:22.938198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.942975Z","iopub.execute_input":"2023-04-01T18:57:22.943437Z","iopub.status.idle":"2023-04-01T18:57:22.969172Z","shell.execute_reply.started":"2023-04-01T18:57:22.943397Z","shell.execute_reply":"2023-04-01T18:57:22.968192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting the train dataframe.","metadata":{}},{"cell_type":"markdown","source":"Lets split the train data into a train and test dataframes. We do this with the module *train_test_split* within sklearn. Within the split we also stratify the data so the proportions of the classes it preserved in both the train and validation set. Even tho the data is well balanced we still wanna stratify our data inorder to overcome any overfitting on a specific label.","metadata":{}},{"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(train_df.iloc[:, 10:], train_df.iloc[:, 0:10], train_size = 0.9, stratify=train_df.iloc[:, 0:10])","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:22.972460Z","iopub.execute_input":"2023-04-01T18:57:22.972766Z","iopub.status.idle":"2023-04-01T18:57:23.648511Z","shell.execute_reply.started":"2023-04-01T18:57:22.972738Z","shell.execute_reply":"2023-04-01T18:57:23.647452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = x_train/255\nx_val = x_val/255","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:23.650030Z","iopub.execute_input":"2023-04-01T18:57:23.650520Z","iopub.status.idle":"2023-04-01T18:57:23.765146Z","shell.execute_reply.started":"2023-04-01T18:57:23.650465Z","shell.execute_reply":"2023-04-01T18:57:23.764058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = pd.concat([x_train, y_train], axis=1).reset_index(drop=True)\nval_dataset = pd.concat([x_val, y_val], axis=1).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:23.766307Z","iopub.execute_input":"2023-04-01T18:57:23.766674Z","iopub.status.idle":"2023-04-01T18:57:23.962141Z","shell.execute_reply.started":"2023-04-01T18:57:23.766645Z","shell.execute_reply":"2023-04-01T18:57:23.961002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:51.720631Z","iopub.execute_input":"2023-04-01T18:57:51.721246Z","iopub.status.idle":"2023-04-01T18:57:51.750075Z","shell.execute_reply.started":"2023-04-01T18:57:51.721194Z","shell.execute_reply":"2023-04-01T18:57:51.748903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking the split","metadata":{}},{"cell_type":"markdown","source":"After we have created our train dataset and validation dataset we then wanna check if we actually did the feature and target matching right. For this check we will take a sample out both of the dataframes and plot these numbers","metadata":{}},{"cell_type":"code","source":"train_sample = train_dataset.sample(5)\nval_sample = val_dataset.sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:52.116215Z","iopub.execute_input":"2023-04-01T18:57:52.116628Z","iopub.status.idle":"2023-04-01T18:57:52.125658Z","shell.execute_reply.started":"2023-04-01T18:57:52.116591Z","shell.execute_reply":"2023-04-01T18:57:52.124524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features, train_label = train_sample.iloc[:, 0:784], train_sample.iloc[:, 784:]\nval_features, val_label = val_sample.iloc[:, 0:784], val_sample.iloc[:, 784:]","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:52.484182Z","iopub.execute_input":"2023-04-01T18:57:52.484575Z","iopub.status.idle":"2023-04-01T18:57:52.492001Z","shell.execute_reply.started":"2023-04-01T18:57:52.484538Z","shell.execute_reply":"2023-04-01T18:57:52.490704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(2, 5, figsize=(10, 5))\n\naxes = axs.ravel()\n\nfig.suptitle('Split check')\n\ntrain_heatmaps = [[np.array(list(row)[1]), list(train_label.iloc[idx, :]).index(max(train_label.iloc[idx, :])), 'train'] for idx, row in enumerate(train_features.iterrows())]\nval_heatmaps = [[np.array(list(row)[1]), list(val_label.iloc[idx, :]).index(max(val_label.iloc[idx, :])), 'validation'] for idx, row in enumerate(val_features.iterrows())]\n\nconcat_list = train_heatmaps + val_heatmaps\n\nfor i, (arr, label, dataset) in enumerate(concat_list):\n    axs = axes[i]\n    \n    sns.heatmap(arr.reshape((28, 28)), ax=axs)\n    axs.set_xlabel(label)\n    axs.set_title(f'{dataset}_sample')\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:52.619927Z","iopub.execute_input":"2023-04-01T18:57:52.620288Z","iopub.status.idle":"2023-04-01T18:57:56.449928Z","shell.execute_reply.started":"2023-04-01T18:57:52.620256Z","shell.execute_reply":"2023-04-01T18:57:56.448722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Pytorch Dataset.","metadata":{}},{"cell_type":"markdown","source":"When we wanna create the pytorch datasets its important to do this after the train test split within the train dataframe. Within this pytorch dataset module we will use data augmentation to see how the model reacts on it. We will also create a option to disable data augmentation","metadata":{}},{"cell_type":"code","source":"class MnistDataset(Dataset):\n    \"\"\"\n    This class is made for the mnist dataset we have currently. This is specifically made for a PyTorch DataLoader.\n    \n    Args:\n        df (pd.DataFrame): A pandas dataframe consisting of either a training dataset or a validation dataset.\n        transform (A.Compose): An Albumentations pipeline object where we define all the transformation methods we are using.\n    \"\"\"\n    \n    def __init__(self, df: pd.DataFrame, transform: A.Compose = None):\n        self.df = df\n        self.transform = transform\n        \n    def __len__(self) -> int:\n        \"\"\"\n        This function returns the length of our dataframe object.\n        \n        Returns:\n            The length of our dataframe.\n        \"\"\"\n        return self.df.shape[0]\n    \n    def __getitem__(self, idx: int) -> Tuple[torch.tensor, torch.tensor]:\n        pixel_series, label = self.df.iloc[idx, :784], self.df.iloc[idx, 784:]\n        \n        pixel_numpy = pixel_series.to_numpy().reshape(28, 28)\n        \n        if self.transform:\n            transformed = self.transform(image=pixel_numpy)[\"image\"]\n            \n        return transformed, np.array(label)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:56.452186Z","iopub.execute_input":"2023-04-01T18:57:56.452693Z","iopub.status.idle":"2023-04-01T18:57:56.464189Z","shell.execute_reply.started":"2023-04-01T18:57:56.452649Z","shell.execute_reply":"2023-04-01T18:57:56.463158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_pytorch_dataset = MnistDataset(df=train_dataset, transform=A.Compose([\n    A.Rotate(limit=20, p=0.5),\n    ToTensorV2(),\n]))\nvalidate_pytorch_dataset = MnistDataset(df=val_dataset, transform=A.Compose([\n    ToTensorV2(),\n]))","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:56.466100Z","iopub.execute_input":"2023-04-01T18:57:56.466557Z","iopub.status.idle":"2023-04-01T18:57:56.476371Z","shell.execute_reply.started":"2023-04-01T18:57:56.466443Z","shell.execute_reply":"2023-04-01T18:57:56.475386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking Pytorch dataset","metadata":{}},{"cell_type":"markdown","source":"Once we have augmentated our training dataset we then can proceed to check out the training dataset to see if everything got augemented right.","metadata":{}},{"cell_type":"code","source":"subset_size = 5\n\ntrain_subset_indices = np.random.choice(len(train_pytorch_dataset), size=subset_size, replace=False)\ntrain_subset_sampler = SubsetRandomSampler(train_subset_indices)\ntrain_subset_loader = torch.utils.data.DataLoader(train_pytorch_dataset, batch_size=1, sampler=train_subset_sampler)\n\nval_subset_indices = np.random.choice(len(validate_pytorch_dataset), size=subset_size, replace=False)\nval_subset_sampler = SubsetRandomSampler(val_subset_indices)\nval_subset_loader = torch.utils.data.DataLoader(validate_pytorch_dataset, batch_size=1, sampler=val_subset_sampler)\n\ntrain_check_set = [[data, np.argmax(target).item(), 'train'] for batch_idx, (data, target) in enumerate(train_subset_loader)]\nval_check_set = [[data, np.argmax(target).item(), 'validate'] for batch_idx, (data, target) in enumerate(val_subset_loader)]\n\nconcat_list = train_check_set + val_check_set\n\nfig, axs = plt.subplots(2, 5, figsize=(10, 5))\n\naxes = axs.ravel()\n\nfig.suptitle('Augmentation split')\n\nfor i, (arr, label, dataset) in enumerate(concat_list):\n\n    axs = axes[i]\n    \n    sns.heatmap(arr.reshape((28, 28)), ax=axs)\n    axs.set_xlabel(label)\n    axs.set_title(dataset)\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:57:56.479767Z","iopub.execute_input":"2023-04-01T18:57:56.480141Z","iopub.status.idle":"2023-04-01T18:58:00.065217Z","shell.execute_reply.started":"2023-04-01T18:57:56.480112Z","shell.execute_reply":"2023-04-01T18:58:00.064052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Pytorch dataloaders.","metadata":{}},{"cell_type":"markdown","source":"After we have done prepping our train and validation datasets into a pytorch dataset, we then can put them inside a pytorch dataloader which we need for batching and fitting the models. We can pull our datasets throught the dataloaders by just passing the datasets to the dataloader function.","metadata":{}},{"cell_type":"code","source":"train_dataloader = DataLoader(train_pytorch_dataset, batch_size=64, shuffle=False)\nval_dataloader = DataLoader(validate_pytorch_dataset, batch_size=64, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:58:00.067122Z","iopub.execute_input":"2023-04-01T18:58:00.067491Z","iopub.status.idle":"2023-04-01T18:58:00.073683Z","shell.execute_reply.started":"2023-04-01T18:58:00.067452Z","shell.execute_reply":"2023-04-01T18:58:00.072481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelling Phase","metadata":{}},{"cell_type":"markdown","source":"## CNN (architecture)","metadata":{}},{"cell_type":"markdown","source":"After we are done setting up the dataloaders we then can start looking at the modelling side of this task. We will first start with our own defined CNN architecture. This architecture will mostly be the baselines for this task. In this architecture we will use the standart batchnormalization.","metadata":{}},{"cell_type":"code","source":"class Simple_CNN(nn.Module):\n    def __init__(self, input_shape: tuple = (1, 28, 28), output_classes: int = 10):\n        super().__init__()\n        \n        self.conv_layer_1 = nn.Sequential(\n            nn.Conv2d(in_channels=input_shape[0], out_channels=32, kernel_size=2),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=2),\n            nn.BatchNorm2d(32),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n        \n        output_h, output_w = self._calculate_output_shape(input_shape = (input_shape[1], input_shape[2]), sequential_block=self.conv_layer_1)\n        \n        self.conv_layer_2 = nn.Sequential(\n            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=2),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.Conv2d(in_channels=64, out_channels=64, kernel_size=2),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=2, stride=2),\n        )\n        \n        output_h, output_w = self._calculate_output_shape(input_shape = (output_h, output_w), sequential_block=self.conv_layer_2)  \n        \n        self.flatten = nn.Flatten()\n        \n        self.fc = nn.Sequential(\n            nn.Linear(output_h * output_w * 64, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.4),\n        )\n        \n        self.fc1 = nn.Sequential(\n            nn.Linear(512, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n        )\n        \n        self.classifier = nn.Linear(256, output_classes)\n        \n    def _calculate_output_shape(self, input_shape: Tuple[int, int], sequential_block: nn.Sequential) -> Tuple[int, int]:\n     \n        output_shape = input_shape\n\n        for layer in sequential_block:\n\n            if isinstance(layer, nn.Conv2d):\n                output_shape = (\n                    (output_shape[0] - layer.kernel_size[0] + 2 * layer.padding[0]) // layer.stride[0] + 1,\n                    (output_shape[1] - layer.kernel_size[1] + 2 * layer.padding[1]) // layer.stride[1] + 1\n                )\n                \n            if isinstance(layer, nn.MaxPool2d):\n\n                output_shape = (\n                    (output_shape[0] - layer.kernel_size) // layer.stride + 1,\n                    (output_shape[1] - layer.kernel_size) // layer.stride + 1\n                )\n\n        return output_shape\n        \n    def forward(self, x):\n        x = self.conv_layer_1(x)\n        x = self.conv_layer_2(x)\n        x = self.flatten(x)\n        x = self.fc(x)\n        x = self.fc1(x)\n        x = self.classifier(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:58:00.076211Z","iopub.execute_input":"2023-04-01T18:58:00.076601Z","iopub.status.idle":"2023-04-01T18:58:00.093988Z","shell.execute_reply.started":"2023-04-01T18:58:00.076563Z","shell.execute_reply":"2023-04-01T18:58:00.092961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN (training)","metadata":{}},{"cell_type":"markdown","source":"Underneath the text you see the fit function thats being used for this model. We train and evaluate the validation set within the same epoch. ","metadata":{}},{"cell_type":"code","source":"def fit(model, loss_fn, optimizer, epochs, train_dataloader, val_dataloader, device):\n    \n    model.train()\n    \n    loss_fn = loss_fn.cuda()\n    \n    train_acc, train_loss, val_acc, val_loss = [], [], [], []\n    \n    for epoch in range(epochs):\n        \n        train_loss_batch, train_acc_batch = 0, 0\n        val_loss_batch, val_acc_batch = 0, 0\n        \n        for inputs, labels in train_dataloader:\n            \n            inputs = inputs.to(device)\n            labels = labels.to(device)\n            \n            inputs = inputs.to(torch.float32)\n            labels = labels.to(torch.float32)\n            \n            # Zero the parameter gradients\n            optimizer.zero_grad()\n            \n            output = model(inputs)\n            \n            loss = loss_fn(output, labels)\n            \n            loss.backward()\n            optimizer.step()\n            \n            train_loss_batch += loss.item()\n            \n            _, predicted = torch.max(output, 1)\n            \n            _, y_true = torch.max(labels, 1)\n            \n            correct = (predicted == y_true).sum().item()\n            \n            train_acc_batch += correct / len(labels)\n            \n        train_average_accuracy = train_acc_batch / len(train_dataloader)\n        train_average_loss = train_loss_batch / len(train_dataloader)\n        \n        train_loss.append(train_average_loss)\n        train_acc.append(train_average_accuracy)\n        \n        model.eval()\n        \n        with torch.no_grad():\n            for inputs, labels in val_dataloader:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n                \n                inputs = inputs.to(torch.float32)\n                labels = labels.to(torch.float32)\n                \n                output = model(inputs)\n                \n                loss = loss_fn(output, labels)\n                \n                val_loss_batch += loss.item()\n                \n                _, predicted = torch.max(output, 1)\n            \n                _, y_true = torch.max(labels, 1)\n                \n                correct = (predicted == y_true).sum().item()\n            \n                val_acc_batch += correct / len(labels)\n                \n            val_average_accuracy = val_acc_batch / len(val_dataloader)\n            val_average_loss = val_loss_batch / len(val_dataloader)\n                \n            val_loss.append(val_average_loss)\n            val_acc.append(val_average_accuracy)\n                    \n    return train_acc, train_loss, val_acc, val_loss","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:58:00.095696Z","iopub.execute_input":"2023-04-01T18:58:00.096111Z","iopub.status.idle":"2023-04-01T18:58:00.110927Z","shell.execute_reply.started":"2023-04-01T18:58:00.096070Z","shell.execute_reply":"2023-04-01T18:58:00.109767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_fn = nn.CrossEntropyLoss()\nepochs = 10\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nnet = Simple_CNN(input_shape=(1, 28, 28), output_classes=10).to(device)\noptimizer = torch.optim.Adam(net.parameters(), lr=0.001)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:58:00.112601Z","iopub.execute_input":"2023-04-01T18:58:00.113125Z","iopub.status.idle":"2023-04-01T18:58:02.838096Z","shell.execute_reply.started":"2023-04-01T18:58:00.113088Z","shell.execute_reply":"2023-04-01T18:58:02.836850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_acc, train_loss, val_acc, val_loss = fit(model=net,\n                                               loss_fn=loss_fn,\n                                               optimizer=optimizer,\n                                               epochs=epochs, \n                                               train_dataloader=train_dataloader, \n                                               val_dataloader=val_dataloader, \n                                               device=device)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T18:58:02.839550Z","iopub.execute_input":"2023-04-01T18:58:02.840213Z","iopub.status.idle":"2023-04-01T19:07:10.593764Z","shell.execute_reply.started":"2023-04-01T18:58:02.840168Z","shell.execute_reply":"2023-04-01T19:07:10.592701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"baseline_eval = pd.DataFrame({\"train_acc\": train_acc, \"train_loss\": train_loss, \"val_acc\": val_acc, \"val_loss\": val_loss})","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:07:10.596622Z","iopub.execute_input":"2023-04-01T19:07:10.596910Z","iopub.status.idle":"2023-04-01T19:07:10.603840Z","shell.execute_reply.started":"2023-04-01T19:07:10.596883Z","shell.execute_reply":"2023-04-01T19:07:10.602720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN (evaluation)","metadata":{}},{"cell_type":"markdown","source":"As we can see we fit the model pretty well considering the amount of data augmentations we do. These metrics could be better tho if we use a cnn with a xgboost added to the last layer. this could mostlikely put us to 1.0 acc","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))\nfig.suptitle('Baseline evaluation')\n\nsns.lineplot(data=baseline_eval, x=baseline_eval.index, y=\"train_acc\", ax=ax1)\nsns.lineplot(data=baseline_eval, x=baseline_eval.index, y=\"val_acc\", ax=ax1)\n\nsns.lineplot(data=baseline_eval, x=baseline_eval.index, y=\"train_loss\", ax=ax2)\nsns.lineplot(data=baseline_eval, x=baseline_eval.index, y=\"val_loss\", ax=ax2)\n\nax1.set_title(\"Accuracy\")\nax1.set_xlabel(\"epoch\")\nax2.set_title(\"Loss\")\nax2.set_xlabel(\"epoch\")\n\nfig.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:07:10.605438Z","iopub.execute_input":"2023-04-01T19:07:10.606349Z","iopub.status.idle":"2023-04-01T19:07:11.159912Z","shell.execute_reply.started":"2023-04-01T19:07:10.606308Z","shell.execute_reply":"2023-04-01T19:07:11.158871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test set","metadata":{}},{"cell_type":"markdown","source":"When we are done evaluating the CNN, we then can proceed to test the model on the test set. before we can actually try out the test set we do need to put it throught the custom dataset and dataloader. after we have done this we can batch the test set up and run it through our model. In the end we will save all the predicts for each image id inside a dataframe which will be exported to a csv by to_csv function within pandas.","metadata":{}},{"cell_type":"markdown","source":"### Test set prep","metadata":{}},{"cell_type":"markdown","source":"To use our test set we first need to transform it to the right shape. we can simply do this by calling our dataset class and insert our test dataframe we have created in the first block.","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:10:59.294418Z","iopub.execute_input":"2023-04-01T19:10:59.295376Z","iopub.status.idle":"2023-04-01T19:10:59.315560Z","shell.execute_reply.started":"2023-04-01T19:10:59.295333Z","shell.execute_reply":"2023-04-01T19:10:59.314406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df / 255","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:01.951878Z","iopub.execute_input":"2023-04-01T19:11:01.952487Z","iopub.status.idle":"2023-04-01T19:11:02.097424Z","shell.execute_reply.started":"2023-04-01T19:11:01.952441Z","shell.execute_reply":"2023-04-01T19:11:02.096434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pytorch_dataset = MnistDataset(df=test_df, transform=A.Compose([\n    ToTensorV2(),\n]))","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:06.008222Z","iopub.execute_input":"2023-04-01T19:11:06.008694Z","iopub.status.idle":"2023-04-01T19:11:06.015151Z","shell.execute_reply.started":"2023-04-01T19:11:06.008651Z","shell.execute_reply":"2023-04-01T19:11:06.014076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(test_pytorch_dataset, batch_size=1, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:06.757452Z","iopub.execute_input":"2023-04-01T19:11:06.757974Z","iopub.status.idle":"2023-04-01T19:11:06.766903Z","shell.execute_reply.started":"2023-04-01T19:11:06.757938Z","shell.execute_reply":"2023-04-01T19:11:06.765614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Test set evaluation","metadata":{}},{"cell_type":"markdown","source":"Underneath the code we can see the test function to test our model based on the test dataframe. We eventually return the ids and predicts for the test set. We save the data inside a dataframe then export it to a csv.","metadata":{}},{"cell_type":"code","source":"def test(model, test_dataloader, device):\n    \n    ids = []\n    preds = []\n    \n    model.eval()\n    \n    with torch.no_grad():\n        for idx, (inputs, labels) in enumerate(test_dataloader):\n            inputs = inputs.to(device)\n            labels = labels.to(device)\n\n            inputs = inputs.to(torch.float32)\n            labels = labels.to(torch.float32)\n\n            output = model(inputs)\n\n            _, predicted = torch.max(output, 1)\n            \n            ids.append(idx + 1)\n            preds.append(predicted.item())\n        \n    return ids, preds","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:10.748981Z","iopub.execute_input":"2023-04-01T19:11:10.749358Z","iopub.status.idle":"2023-04-01T19:11:10.757058Z","shell.execute_reply.started":"2023-04-01T19:11:10.749324Z","shell.execute_reply":"2023-04-01T19:11:10.755830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids, pred = test(model=net, test_dataloader=test_dataloader, device=device)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:11.699292Z","iopub.execute_input":"2023-04-01T19:11:11.699687Z","iopub.status.idle":"2023-04-01T19:11:58.065994Z","shell.execute_reply.started":"2023-04-01T19:11:11.699653Z","shell.execute_reply":"2023-04-01T19:11:58.064931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_df = pd.DataFrame({\"ImageId\": ids, \"Label\": pred})","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:11:58.067916Z","iopub.execute_input":"2023-04-01T19:11:58.068268Z","iopub.status.idle":"2023-04-01T19:11:58.086884Z","shell.execute_reply.started":"2023-04-01T19:11:58.068231Z","shell.execute_reply":"2023-04-01T19:11:58.085927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Saving it to csv","metadata":{}},{"cell_type":"markdown","source":"At last we save the prediction on our test set to a csv. we can do this by calling the 'to_csv' function within pandas.","metadata":{}},{"cell_type":"code","source":"preds_df.to_csv('submissions.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-01T19:12:13.966998Z","iopub.execute_input":"2023-04-01T19:12:13.967373Z","iopub.status.idle":"2023-04-01T19:12:14.001307Z","shell.execute_reply.started":"2023-04-01T19:12:13.967339Z","shell.execute_reply":"2023-04-01T19:12:14.000265Z"},"trusted":true},"execution_count":null,"outputs":[]}]}