{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \n#Pytorch stuff\nimport torch\nimport torch.nn.functional as F\nfrom torch.utils.data import random_split, DataLoader, Dataset\nfrom torch import nn\n\n#Visualisation\nimport tqdm\nimport matplotlib.pyplot as plt\nfrom PIL import Image\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-26T23:46:58.365847Z","iopub.execute_input":"2021-07-26T23:46:58.366241Z","iopub.status.idle":"2021-07-26T23:46:59.662065Z","shell.execute_reply.started":"2021-07-26T23:46:58.366146Z","shell.execute_reply":"2021-07-26T23:46:59.661121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the Training Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:28:53.446844Z","iopub.execute_input":"2021-07-26T23:28:53.447276Z","iopub.status.idle":"2021-07-26T23:28:57.077018Z","shell.execute_reply.started":"2021-07-26T23:28:53.447237Z","shell.execute_reply":"2021-07-26T23:28:57.076205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use dataframe to get x and y values for training","metadata":{}},{"cell_type":"code","source":"x = torch.tensor(df.loc[:,'pixel0':'pixel783'].values)\ny = torch.tensor(df.loc[:,'label'].values)\nimgx = np.array(x.reshape((-1,28,28)), dtype = np.uint8)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:07.927128Z","iopub.execute_input":"2021-07-26T23:26:07.927494Z","iopub.status.idle":"2021-07-26T23:26:08.124868Z","shell.execute_reply.started":"2021-07-26T23:26:07.927456Z","shell.execute_reply":"2021-07-26T23:26:08.123963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.fromarray(imgx[3], mode='L')\nimg","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:08.127466Z","iopub.execute_input":"2021-07-26T23:26:08.128003Z","iopub.status.idle":"2021-07-26T23:26:08.145374Z","shell.execute_reply.started":"2021-07-26T23:26:08.12796Z","shell.execute_reply":"2021-07-26T23:26:08.14442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating a baseline model\nA simple baseline model would be taking the average pixel brightness for each digit, then compare an image's average brightness to recorded values to classify the image.\n\nWe will make this model general enough to account for varying numbers of choices for different digits.","metadata":{}},{"cell_type":"code","source":"#sorts data into different labels\nlabels = list(set(df[\"label\"])) #gives enumeration for the labels\ndef label_map(index):\n    return labels[index]\n\nsorted_data = []\nfor label in labels:\n    sorted_data.append(torch.tensor(df.loc[df['label'] == label].loc[:,'pixel0':'pixel783'].values))","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:34:34.324327Z","iopub.execute_input":"2021-07-26T23:34:34.32469Z","iopub.status.idle":"2021-07-26T23:34:34.726659Z","shell.execute_reply.started":"2021-07-26T23:34:34.324654Z","shell.execute_reply":"2021-07-26T23:34:34.725777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"A nice way to display tensors","metadata":{}},{"cell_type":"code","source":"def display_tns(t):\n    ldf = pd.DataFrame(np.array(t.reshape(28,28)))\n    return ldf.style.set_properties(**{'font-size':'5pt'}).background_gradient('Greys',axis = None)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:08.534958Z","iopub.execute_input":"2021-07-26T23:26:08.53533Z","iopub.status.idle":"2021-07-26T23:26:08.541084Z","shell.execute_reply.started":"2021-07-26T23:26:08.535291Z","shell.execute_reply":"2021-07-26T23:26:08.53929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Generates a heatmap for each of the digits","metadata":{}},{"cell_type":"code","source":"averages = torch.tensor(list(map(lambda x: np.array(x).mean((0)),sorted_data)))\ndisplay_tns(averages[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:08.542453Z","iopub.execute_input":"2021-07-26T23:26:08.542819Z","iopub.status.idle":"2021-07-26T23:26:08.781825Z","shell.execute_reply.started":"2021-07-26T23:26:08.542781Z","shell.execute_reply":"2021-07-26T23:26:08.78095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We define the prediction functions and determine the models accuracy","metadata":{}},{"cell_type":"code","source":"def classification_losses(s):\n    if len(s.shape) == 2:\n        t = s\n    else:\n        t = s.unsqueeze_(0)\n    losses = torch.empty(0,t.shape[0])\n    for average in averages:\n        losses = torch.cat((losses,(t-average).abs().mean((-1)).unsqueeze_(0)),0)\n    return losses\n\ndef predict(t):\n    return torch.tensor([label_map(a) for a in torch.argmin(classification_losses(t), dim = 0)])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:08.784227Z","iopub.execute_input":"2021-07-26T23:26:08.784793Z","iopub.status.idle":"2021-07-26T23:26:08.792175Z","shell.execute_reply.started":"2021-07-26T23:26:08.784756Z","shell.execute_reply":"2021-07-26T23:26:08.791391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classification_losses(x).shape, predict(x), predict(x[1])","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:08.794Z","iopub.execute_input":"2021-07-26T23:26:08.794501Z","iopub.status.idle":"2021-07-26T23:26:19.047076Z","shell.execute_reply.started":"2021-07-26T23:26:08.794464Z","shell.execute_reply":"2021-07-26T23:26:19.046027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy = torch.tensor([a == b for (a,b) in zip(predict(x),y)]).float().mean()\naccuracy","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:19.048796Z","iopub.execute_input":"2021-07-26T23:26:19.049163Z","iopub.status.idle":"2021-07-26T23:26:24.78768Z","shell.execute_reply.started":"2021-07-26T23:26:19.04912Z","shell.execute_reply":"2021-07-26T23:26:24.786828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So now we have a baseline model with which to compare our future modelling attempts.\nWe can also use this model to make a submission.","metadata":{}},{"cell_type":"code","source":"testdf = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ntestx = torch.tensor(testdf.loc[:,'pixel0':'pixel783'].values, dtype = torch.float)\npredictions_baseline = predict(testx)\ndata_baseline = {\"ImageID\":np.arange(1,len(predictions_baseline)+1),\n                 \"Label\":predictions_baseline}\nsubmission_baseline = pd.DataFrame(data_baseline,columns = [\"ImageID\",\"Label\"])\nsubmission_baseline.to_csv(\"baseline.csv\",index = False)\nsubmission_baseline","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:46:59.663718Z","iopub.execute_input":"2021-07-26T23:46:59.664108Z","iopub.status.idle":"2021-07-26T23:47:02.111339Z","shell.execute_reply.started":"2021-07-26T23:46:59.664067Z","shell.execute_reply":"2021-07-26T23:47:02.109389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using machine learning to do better\nFirstly we're gonna use a simple neural network to try and improve on the 0.65 or so accuracy. Let's first prepare the data by splitting it into validation and training data.","metadata":{}},{"cell_type":"code","source":"#Typically Dataset would be taken from a saved file rather than stored dynamically, but this is just an experiment\nclass CustomDatasetMNIST(Dataset):\n    def __init__(self, data_tensor):\n        self.x = data_tensor[:,1:]\n        self.y = data_tensor[:,0]\n        \n    def __len__(self):\n        return len(self.y)\n    \n    def __getitem__(self,idx):\n        x = self.x[idx]/255 #So all data points lie in [0,1]\n        y = int(self.y[idx])\n        return x,y\n        ","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:29:09.718589Z","iopub.execute_input":"2021-07-26T23:29:09.718954Z","iopub.status.idle":"2021-07-26T23:29:09.724793Z","shell.execute_reply.started":"2021-07-26T23:29:09.718922Z","shell.execute_reply":"2021-07-26T23:29:09.723749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train, valid = random_split(CustomDatasetMNIST(torch.tensor(df.values, dtype = torch.float)),[36000,6000])\ntrain_loader = DataLoader(train, batch_size = 32)\nvalid_loader = DataLoader(valid, batch_size = 32)\ncriterion = nn.CrossEntropyLoss()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:32:13.849083Z","iopub.execute_input":"2021-07-26T23:32:13.851903Z","iopub.status.idle":"2021-07-26T23:32:14.007017Z","shell.execute_reply.started":"2021-07-26T23:32:13.851853Z","shell.execute_reply":"2021-07-26T23:32:14.005918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First is going to be a very simple neural network, with two hidden layers using RELU as a non-linear function.","metadata":{}},{"cell_type":"code","source":"class SimpleNetworkMNIST(nn.Module):\n    def __init__(self):\n        super(SimpleNetworkMNIST,self).__init__()\n        self.l1 = nn.Linear(784,400)\n        self.l2 = nn.Linear(400,100)\n        self.l3 = nn.Linear(100,10)\n    \n    def forward(self,x):\n        x = F.normalize(x)\n        x = F.relu(self.l1(x))\n        x = F.relu(self.l2(x))\n        x = self.l3(x)\n        return x\n        \nsimple_model = SimpleNetworkMNIST()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:30.514802Z","iopub.execute_input":"2021-07-26T23:26:30.515198Z","iopub.status.idle":"2021-07-26T23:26:30.527304Z","shell.execute_reply.started":"2021-07-26T23:26:30.515145Z","shell.execute_reply":"2021-07-26T23:26:30.526391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We also want to describe the loss function we will be using and the optimisation method. For our loss function we will use cross-entropy loss, which is standard for multi-label classification, and for our optimiser, stochastic gradient descent","metadata":{}},{"cell_type":"code","source":"simple_optimiser = torch.optim.SGD(simple_model.parameters(), lr = 0.1) #lr denotes a learning rate","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:29:53.370694Z","iopub.execute_input":"2021-07-26T23:29:53.371065Z","iopub.status.idle":"2021-07-26T23:29:53.386094Z","shell.execute_reply.started":"2021-07-26T23:29:53.371031Z","shell.execute_reply":"2021-07-26T23:29:53.384819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Then we can train the model, while keeping an eye on the metrics:","metadata":{}},{"cell_type":"code","source":"epochs = 2\nfor epoch in range(epochs):\n    #Train over the epoch\n    simple_model.train()\n    running_loss = 0.0\n    for xs, ys in train_loader:\n        simple_optimiser.zero_grad()\n        prediction_ys = simple_model(xs)\n        loss = criterion(prediction_ys,ys)\n        loss.backward()\n        simple_optimiser.step() #Note this works as applying loss.backward works on the parameters of the model which is the argument for the optimiser\n        running_loss += loss.item()\n    train_loss = running_loss/len(train_loader)\n    #Get the validation loss\n    simple_model.eval()\n    running_loss = 0.0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for xs,ys in valid_loader:\n            prediction_ys = simple_model(xs)\n            loss = criterion(prediction_ys,ys)\n            running_loss += loss.item()\n            total += ys.size(0)\n            correct += (prediction_ys.max(1)[1]).eq(ys).sum().item()\n    valid_loss = running_loss/len(valid_loader)\n    accuracy = 100.0*correct/total\n    print(f\"Epoch: {epoch+1}/{epochs}:\\nTraining loss: {train_loss}, Validation loss: {valid_loss}, Accuracy: {accuracy}\")","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:30.537442Z","iopub.execute_input":"2021-07-26T23:26:30.537801Z","iopub.status.idle":"2021-07-26T23:26:39.338466Z","shell.execute_reply.started":"2021-07-26T23:26:30.537765Z","shell.execute_reply":"2021-07-26T23:26:39.337397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Obviously this model could be trained for longer and with more sophisticated methods, which we will implement later, but let's generate a submission with this initial model (although take note that this model has a much higher accuracy than that of the baseline model showing us that the use of machine learning has been useful).","metadata":{}},{"cell_type":"code","source":"simple_model.eval()\npredictions_simple = simple_model(testx).max(1)[1]\ndata_simple = {\"ImageID\":np.arange(1,len(predictions_simple)+1),\n                 \"Label\":predictions_simple}\nsubmission_simple = pd.DataFrame(data_simple,columns = [\"ImageID\",\"Label\"])\nsubmission_simple.to_csv(\"simple.csv\",index = False)\nsubmission_simple","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:39.339981Z","iopub.execute_input":"2021-07-26T23:26:39.340597Z","iopub.status.idle":"2021-07-26T23:26:39.745659Z","shell.execute_reply.started":"2021-07-26T23:26:39.340549Z","shell.execute_reply":"2021-07-26T23:26:39.744659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Interpretation\nAlthough we have metrics as we train our AI, we might want to look at some of the details of the AI. We can use visualisation methods such as:\n* Confusion matrices\n* Top losses\n\nBelow we can generate a confusion matrix by tallying the times when data with some label was predicted as some label","metadata":{}},{"cell_type":"code","source":"confusion_matrix = np.zeros((len(labels),len(labels)),dtype = np.uint16)\nMNISTdata = torch.tensor(df.values, dtype = torch.float)\nsimple_model_out = simple_model(MNISTdata[:,1:])\ntargets = MNISTdata[:,0]\n\n#Could potentially also exclusively consider validation data to account for potential overfitting\nfor prediction,actual in zip(simple_model_out.max(1)[1],targets):\n    confusion_matrix[prediction,actual.int()] += 1\nconfusion_matrix","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:39.747038Z","iopub.execute_input":"2021-07-26T23:26:39.74743Z","iopub.status.idle":"2021-07-26T23:26:41.241624Z","shell.execute_reply.started":"2021-07-26T23:26:39.747375Z","shell.execute_reply":"2021-07-26T23:26:41.240597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can display this information a bit more clearly using matplotlib:","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (10,10))\n\nmax_val = len(labels)\n\nfor i in range(max_val):\n    for j in range(max_val):\n        ax.text(i, j, str(confusion_matrix[i,j]), va='center', ha='center')\n\nplt.title(\"Confusion Matrix\")\nplt.xlabel(\"Predicted digit\")\nplt.ylabel(\"Actual digit\")\nplt.xticks(np.arange(0,max_val),labels)\nplt.yticks(np.arange(0,max_val),labels)\nax.tick_params(grid_alpha = 0)\n\n\nax.set_xlim(-0.5, max_val-0.5)\nax.set_ylim(-0.5, max_val-0.5)\nax.imshow(confusion_matrix, cmap=plt.cm.Blues, origin = 'lower', aspect = 'equal')\nax.grid()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:41.242982Z","iopub.execute_input":"2021-07-26T23:26:41.243371Z","iopub.status.idle":"2021-07-26T23:26:41.893582Z","shell.execute_reply.started":"2021-07-26T23:26:41.243332Z","shell.execute_reply":"2021-07-26T23:26:41.892738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Looking at this data we see some really interesting and expected flaws in our model, for example it frequently mixes up (2,7) and (4,9) as they are similar looking. We can also see its accuracy in very different looking digits, such as between (3,4) and (2,5). We can also take some of the predictions that resulted in the highest losses to inspect what aspects of these particular examples make them so hard to classify.","metadata":{}},{"cell_type":"code","source":"top_no = 5\nsorted_losses, indices = nn.CrossEntropyLoss(reduction = 'none')(simple_model_out,targets.long()).sort(descending = True)\n\n\nfor i in range(top_no):\n    loss, actual, predicted = sorted_losses[i], targets[indices[i]].int(), simple_model_out[indices[i]].max(0)[1]\n    data = MNISTdata[indices[i],1:]\n    \n    plt.figure()\n    plt.title(f'Loss: {loss}, Predicted: {predicted}, Actual: {actual}')\n    plt.imshow(np.array(data.reshape(28,28)),cmap = plt.cm.gray)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:41.894898Z","iopub.execute_input":"2021-07-26T23:26:41.895282Z","iopub.status.idle":"2021-07-26T23:26:42.677701Z","shell.execute_reply.started":"2021-07-26T23:26:41.895241Z","shell.execute_reply":"2021-07-26T23:26:42.675606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see in these top losses are examples of particularly poorly written digits, and in each case (except maybe that 3), you can see what the model predicted in the image. Depending on how this model is trained you may also find erroneous data (there is a 7 mislabelled as a 4 in this dataset), and this highlights a use of this visualisation; removing bad data to aid in the AI's learning process.","metadata":{}},{"cell_type":"markdown","source":"# Improving our initial model\nSo we're now going to try improving the model by using a convolutional neural network. The main benefit of this is that the AI can now use the spatial aspects of the image, which will aid in it finding patterns in images (and with some tricks we can see what these patterns are later on).\n\nThe nn.Conv2D class has parameters:\n* **in_channels** - number of input planes\n* **out_channels** - number of output planes\n* **kernel_size** - size of convolution (also specifies number of weights and biases)\n* **stride** - int or tuple describing how far the convolution is shifted over for each evaluation of the convolution\n* **padding** - empty pixels around the image (aids in evaluating pixels around the perimeter)\n* **dilation** - describes distance between evaluation points in the kernel (e.g. dilation = 2 on a 3x3 kernel means evaluation of the 9 friends spread out in a 5x5 area)\n* **groups** - splits connections between input and output layers (both must be divisible by groups value, e.g. groups = 2 is similar to having 2 convolution layers in tandem)\n* **padding_mode** - how the padding pixels are filled, can be one of a few options, default 'zeros'\n* **device**\n* **dtype**","metadata":{}},{"cell_type":"code","source":"class ConvolutionMNIST(nn.Module):\n    def __init__(self):\n        super(ConvolutionMNIST,self).__init__()\n        self.cnn_layers = nn.Sequential(\n            nn.Conv2d(1, 4, kernel_size = 3, padding = 1),\n            #BatchNorm normalises the data over the features (planes) seperately over the batch and pixels\n            nn.BatchNorm2d(4),             \n            nn.ReLU(),\n            #Takes maximum value in a 2 by 2 area of convolution output\n            nn.MaxPool2d(kernel_size=2,stride = 2), \n            \n            nn.Conv2d(4,4,kernel_size = 3, padding = 1),\n            nn.BatchNorm2d(4),\n            nn.ReLU(),\n        )\n        \n        #The size of the output planes above is 14 by 14 as size is preserved by conv layer with padding but halved by max pooling\n        self.l1 = nn.Linear(4 * 14 * 14,10)\n        \n    def forward(self,x):\n        x = x.reshape((-1,1,28,28))\n        x = self.cnn_layers(x)\n        x = torch.flatten(x, 1)\n        x = self.l1(x)\n        return x\n    \nconv_model = ConvolutionMNIST()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:42.679467Z","iopub.execute_input":"2021-07-26T23:26:42.679922Z","iopub.status.idle":"2021-07-26T23:26:42.69357Z","shell.execute_reply.started":"2021-07-26T23:26:42.679868Z","shell.execute_reply":"2021-07-26T23:26:42.692151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.path.exists(\"/kaggle//////working\")","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:42.695671Z","iopub.execute_input":"2021-07-26T23:26:42.696509Z","iopub.status.idle":"2021-07-26T23:26:42.707931Z","shell.execute_reply.started":"2021-07-26T23:26:42.69644Z","shell.execute_reply":"2021-07-26T23:26:42.707136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Just so we don't have to repeat code, let's define a function to train the models, we'll also add the ability to save models after epochs where the validation losses improved.","metadata":{}},{"cell_type":"code","source":"def train_model(model, epochs, train_loader, valid_loader, optimiser, criterion, print_output = True, save_models = False, model_path = None, save_as = \"model\"):\n    if save_models:\n        if not model_path:\n            raise ValueError(\"If save_models is True then model_path must be set\")\n        if not os.path.exists(model_path):\n            os.mkdir(model_path)\n    train_losses = []\n    valid_losses = []\n    accuracies = []\n    for epoch in range(epochs):\n        model.train()\n        running_loss = 0.0\n        #Some variables have been renamed for clarity\n        for data, targets in train_loader:\n            optimiser.zero_grad()\n            scores = model(data)\n            loss = criterion(scores,targets)\n            loss.backward()\n            optimiser.step() #Note this works as applying loss.backward works on the parameters of the model which is the argument for the optimiser\n            running_loss += loss.item()\n        train_loss = running_loss/len(train_loader)\n\n        model.eval()\n        running_loss = 0.0\n        correct = 0\n        total = 0\n        with torch.no_grad():\n            for data, targets in valid_loader:\n                scores = model(data)\n                loss = criterion(scores,targets)\n                running_loss += loss.item()\n                total += targets.size(0)\n                correct += (scores.max(1)[1]).eq(targets).sum().item()\n        valid_loss = running_loss/len(valid_loader)\n        accuracy = 100.0*correct/total\n        \n        train_losses.append(train_loss)\n        valid_losses.append(valid_loss)\n        accuracies.append(accuracy)\n        \n        if print_output:\n            print(f\"Epoch: {epoch+1}/{epochs}:\\nTraining loss: {train_loss}, Validation loss: {valid_loss}, Accuracy: {accuracy}\")\n        if save_models:\n            torch.save(model.state_dict(), model_path + f\"/{save_as}_e{epoch+1}\")\n    #For display\n    return train_losses, valid_losses, accuracies","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:32:25.04363Z","iopub.execute_input":"2021-07-26T23:32:25.043997Z","iopub.status.idle":"2021-07-26T23:32:25.060532Z","shell.execute_reply.started":"2021-07-26T23:32:25.043963Z","shell.execute_reply":"2021-07-26T23:32:25.059669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conv_optimiser = torch.optim.SGD(conv_model.parameters(), lr = 0.01)\ntrain_model(conv_model,2,train_loader,valid_loader,conv_optimiser,criterion)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:26:42.724542Z","iopub.execute_input":"2021-07-26T23:26:42.725159Z","iopub.status.idle":"2021-07-26T23:27:02.537256Z","shell.execute_reply.started":"2021-07-26T23:26:42.72512Z","shell.execute_reply":"2021-07-26T23:27:02.536328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Using our trained model we can generate a submission:","metadata":{}},{"cell_type":"code","source":"conv_model.eval()\npredictions_conv = conv_model(testx).max(1)[1]\ndata_conv = {\"ImageID\":np.arange(1,len(predictions_conv)+1),\n                 \"Label\":predictions_conv}\nsubmission_conv = pd.DataFrame(data_conv,columns = [\"ImageID\",\"Label\"])\nsubmission_conv.to_csv(\"conv.csv\",index = False)\nsubmission_conv","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:27:02.538524Z","iopub.execute_input":"2021-07-26T23:27:02.538907Z","iopub.status.idle":"2021-07-26T23:27:05.479007Z","shell.execute_reply.started":"2021-07-26T23:27:02.538856Z","shell.execute_reply":"2021-07-26T23:27:05.477971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see though that this results in a similar result to what we were getting previously. We can improve the model further by adding more layers to the model:","metadata":{}},{"cell_type":"code","source":"class DeepConvolutionMNIST(nn.Module):\n    def __init__(self):\n        super(DeepConvolutionMNIST,self).__init__()\n        self.cnn_layers = nn.Sequential(\n            nn.Conv2d(1, 16, kernel_size = 3, padding = 1),\n            nn.BatchNorm2d(16),             \n            nn.ReLU(),\n            \n            nn.Conv2d(16 ,16 , kernel_size = 3, padding = 1),\n            nn.BatchNorm2d(16),\n            nn.ReLU(),\n            \n            nn.MaxPool2d(kernel_size=2, stride = 2), \n            \n            nn.Conv2d(16, 8, kernel_size = 3, padding = 1),\n            nn.BatchNorm2d(8),\n            nn.ReLU(),\n            \n            nn.MaxPool2d(kernel_size=2, stride = 2),\n            \n            nn.Conv2d(8, 4, kernel_size = 3, padding = 1),\n            nn.BatchNorm2d(4),\n            nn.ReLU(),\n        )\n        \n        self.l1 = nn.Linear(4 * 7 * 7,10)\n        \n    def forward(self,x):\n        x = x.reshape((-1,1,28,28))\n        x = self.cnn_layers(x)\n        x = torch.flatten(x, 1)\n        x = self.l1(x)\n        return x\n    \ndeepconv_model = DeepConvolutionMNIST()","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:32:31.063871Z","iopub.execute_input":"2021-07-26T23:32:31.064251Z","iopub.status.idle":"2021-07-26T23:32:31.078025Z","shell.execute_reply.started":"2021-07-26T23:32:31.064215Z","shell.execute_reply":"2021-07-26T23:32:31.077083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deepconv_optimiser = torch.optim.SGD(deepconv_model.parameters(), lr = 0.01)\ntrain_model(deepconv_model,2,train_loader,valid_loader,deepconv_optimiser,criterion)","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:32:33.724691Z","iopub.execute_input":"2021-07-26T23:32:33.725042Z","iopub.status.idle":"2021-07-26T23:33:32.558064Z","shell.execute_reply.started":"2021-07-26T23:32:33.72501Z","shell.execute_reply":"2021-07-26T23:33:32.557247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deepconv_model.eval()\npredictions_deepconv = deepconv_model(testx).max(1)[1]\ndata_deepconv = {\"ImageID\":np.arange(1,len(predictions_deepconv)+1),\n                 \"Label\":predictions_deepconv}\nsubmission_deepconv = pd.DataFrame(data_deepconv,columns = [\"ImageID\",\"Label\"])\nsubmission_deepconv.to_csv(\"deepconv.csv\",index = False)\nsubmission_deepconv","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:34:09.345009Z","iopub.execute_input":"2021-07-26T23:34:09.345381Z","iopub.status.idle":"2021-07-26T23:34:21.791156Z","shell.execute_reply.started":"2021-07-26T23:34:09.345348Z","shell.execute_reply":"2021-07-26T23:34:21.790163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"img = Image.fromarray(np.array(testx[2], dtype = np.uint8).reshape(28,28), mode='L')\nimg","metadata":{"execution":{"iopub.status.busy":"2021-07-26T23:48:18.991845Z","iopub.execute_input":"2021-07-26T23:48:18.992188Z","iopub.status.idle":"2021-07-26T23:48:19.017607Z","shell.execute_reply.started":"2021-07-26T23:48:18.992156Z","shell.execute_reply":"2021-07-26T23:48:19.016871Z"},"trusted":true},"execution_count":null,"outputs":[]}]}