{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-15T13:05:46.740127Z","iopub.execute_input":"2023-04-15T13:05:46.740641Z","iopub.status.idle":"2023-04-15T13:05:46.758029Z","shell.execute_reply.started":"2023-04-15T13:05:46.740596Z","shell.execute_reply":"2023-04-15T13:05:46.756513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader, random_split","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:46.760812Z","iopub.execute_input":"2023-04-15T13:05:46.761239Z","iopub.status.idle":"2023-04-15T13:05:47.680687Z","shell.execute_reply.started":"2023-04-15T13:05:46.761203Z","shell.execute_reply":"2023-04-15T13:05:47.679634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this notebook I will build my first Neural Net using Pytorch. For simplicity it will only consist of fully conncect layers.\n\n- Much of this notebook is based on chapter 4 of the Fastbook (https://course.fast.ai/Resources/book.html), which I can only recommend.\n- If you want to get familar with PyTorch quickly, check out this tutorial: https://pytorch.org/tutorials/beginner/deep_learning_60min_blitz.html","metadata":{}},{"cell_type":"markdown","source":"## 1. The Data\n\nThe dataset consists out 28 by 28 pixel images.","metadata":{}},{"cell_type":"code","source":"# Load the dataframes\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:47.682039Z","iopub.execute_input":"2023-04-15T13:05:47.682772Z","iopub.status.idle":"2023-04-15T13:05:50.891307Z","shell.execute_reply.started":"2023-04-15T13:05:47.682727Z","shell.execute_reply":"2023-04-15T13:05:50.890235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform the data\ny_train = df_train['label']\nX_train = np.array(df_train.drop('label', axis=1)).reshape(-1, 28, 28)\nX_test = np.array(df_test)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:19:37.351472Z","iopub.execute_input":"2023-04-15T13:19:37.351869Z","iopub.status.idle":"2023-04-15T13:19:37.587736Z","shell.execute_reply.started":"2023-04-15T13:19:37.351838Z","shell.execute_reply":"2023-04-15T13:19:37.586523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We should transform our images to be between -1 and 1.","metadata":{}},{"cell_type":"code","source":"print(X_train.min(), X_train.max())\nX_train = (2/255)*X_train - 1\nprint(X_train.min(), X_train.max())","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.136131Z","iopub.execute_input":"2023-04-15T13:05:51.136615Z","iopub.status.idle":"2023-04-15T13:05:51.361634Z","shell.execute_reply.started":"2023-04-15T13:05:51.136576Z","shell.execute_reply":"2023-04-15T13:05:51.360210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# To prevent errors the data has to be converted to float32\nX_train = X_train.astype('float32')\nX_test = X_test.astype('float32')\n\n# The target has to Integer for our loss function (https://stackoverflow.com/questions/60440292/runtimeerror-expected-scalar-type-long-but-found-float)\ny_train =  y_train.astype(int)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:20:21.063094Z","iopub.execute_input":"2023-04-15T13:20:21.063871Z","iopub.status.idle":"2023-04-15T13:20:21.162848Z","shell.execute_reply.started":"2023-04-15T13:20:21.063834Z","shell.execute_reply":"2023-04-15T13:20:21.161814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a sample\nindex = 2\nplt.imshow(X_train[index])\nplt.show()\nprint('The label of this picture is: ' + str(y_train[index]))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.457350Z","iopub.execute_input":"2023-04-15T13:05:51.457770Z","iopub.status.idle":"2023-04-15T13:05:51.675997Z","shell.execute_reply.started":"2023-04-15T13:05:51.457728Z","shell.execute_reply":"2023-04-15T13:05:51.674736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Before we can contiune with our model we have to convert our data into DataLoaders that PyTorch can use for training.","metadata":{}},{"cell_type":"code","source":"# Generate dataset\ndataset = TensorDataset(torch.tensor(X_train), torch.tensor(y_train))\n\n# split in train and validation set\ng = torch.Generator().manual_seed(42)\ntrain, val = random_split(dataset, [0.8, 0.2], generator=g)\ntrain_loader = DataLoader(train)\nval_loader = DataLoader(val)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.677744Z","iopub.execute_input":"2023-04-15T13:05:51.678483Z","iopub.status.idle":"2023-04-15T13:05:51.910543Z","shell.execute_reply.started":"2023-04-15T13:05:51.678442Z","shell.execute_reply":"2023-04-15T13:05:51.909501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. The Model\n\nNow we define a simple fully connected Neutal Network. We have two hidden layers with 16 nodes each and add our nonlinearites by using the ReLU function. Our output layer has 10 units (one for each digit).\n\nThe softmax layer was left out on purpose as it will be applied in the loss-function (https://stackoverflow.com/questions/55675345/should-i-use-softmax-as-output-when-using-cross-entropy-loss-in-pytorch).","metadata":{}},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\nclass Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.fc1 = nn.Linear(28*28, 16)\n        self.fc2 = nn.Linear(16, 16)\n        self.fc3 = nn.Linear(16, 10)\n        \n    def forward(self, x):\n        x = torch.flatten(x, 1) # flatten everything except the batch dimension (1 is the start_dims, so we don't flatten along the 0 dim which is the batch)\n        x = F.relu(self.fc1(x))\n        x = F.relu(self.fc2(x))\n        x = F.relu(self.fc3(x))\n        return x\n        \nnet = Net()","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.911930Z","iopub.execute_input":"2023-04-15T13:05:51.912608Z","iopub.status.idle":"2023-04-15T13:05:51.922194Z","shell.execute_reply.started":"2023-04-15T13:05:51.912569Z","shell.execute_reply":"2023-04-15T13:05:51.921036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. Defining a Loss function and optimizer \n\n- As a loss function we will be using Classification Cross-Entropy Loss. A good explaination of this loss function can be found here: https://towardsdatascience.com/cross-entropy-loss-function-f38c4ec8643e\n- As a optimizer we will be using Stochastic Gradient Descent.\n\n","metadata":{}},{"cell_type":"code","source":"import torch.optim as optim\n\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) # experiement with the learning rate, if train for longer it is often useful to decreae the learning rate after a few epochs","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.923560Z","iopub.execute_input":"2023-04-15T13:05:51.924103Z","iopub.status.idle":"2023-04-15T13:05:51.931945Z","shell.execute_reply.started":"2023-04-15T13:05:51.924067Z","shell.execute_reply":"2023-04-15T13:05:51.930815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#data, labels = next(iter(train_loader))","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.933346Z","iopub.execute_input":"2023-04-15T13:05:51.934332Z","iopub.status.idle":"2023-04-15T13:05:51.943247Z","shell.execute_reply.started":"2023-04-15T13:05:51.934295Z","shell.execute_reply":"2023-04-15T13:05:51.941625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#net(data)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:05:51.945320Z","iopub.execute_input":"2023-04-15T13:05:51.946304Z","iopub.status.idle":"2023-04-15T13:05:51.958272Z","shell.execute_reply.started":"2023-04-15T13:05:51.946268Z","shell.execute_reply":"2023-04-15T13:05:51.957165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. Training the network\n\nWe will now train our network. A detailed explained of all the steps can be found in fastbook chapter 4 (see above).","metadata":{}},{"cell_type":"code","source":"for epoch in range(5): # a epoch is one iteration over all the data\n    \n        running_loss = 0.0\n        for i, data in enumerate(train_loader, 0):\n            # get the inputs (this is a batch, so lists of images & labels)\n            inputs, labels = data\n            \n            # reset the gradient\n            optimizer.zero_grad()\n            \n            # forward + backward + optimize\n            outputs = net(inputs) # forward\n            loss = criterion(outputs, labels) # loss\n            loss.backward() # backward\n            optimizer.step() # update\n            \n            # print statistics\n            running_loss += loss.item()\n            if i % 2000 == 1999: # print every 2000 mini-batches\n                print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 2000:.3f}')\n                running_loss = 0.0\n\nprint('Training finished')","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:11:42.609683Z","iopub.execute_input":"2023-04-15T13:11:42.610168Z","iopub.status.idle":"2023-04-15T13:14:13.481435Z","shell.execute_reply.started":"2023-04-15T13:11:42.610127Z","shell.execute_reply":"2023-04-15T13:14:13.480248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5. Testing on the validation set\n\nNow we compute the accuracy on the validation set.","metadata":{}},{"cell_type":"code","source":"correct = 0\ntotal = 0\n\nwith torch.no_grad():\n    for data in train_loader:\n        images, labels = data\n        output = net(images)\n        _, predicted = torch.max(output.data, 1)\n        total += labels.size(0)\n        correct += (predicted == labels).sum().item()\n        \nprint(correct/total)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:20:30.539216Z","iopub.execute_input":"2023-04-15T13:20:30.539615Z","iopub.status.idle":"2023-04-15T13:20:38.545530Z","shell.execute_reply.started":"2023-04-15T13:20:30.539583Z","shell.execute_reply":"2023-04-15T13:20:38.544405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. Submitting the solution\n\nIt is now time to submit our solution. Before we can do this we need to predict our test data.\n","metadata":{}},{"cell_type":"code","source":"_, test_predictions = torch.max(net(torch.tensor(X_test)),1)\ntest_predictions","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:21:57.399906Z","iopub.execute_input":"2023-04-15T13:21:57.400839Z","iopub.status.idle":"2023-04-15T13:21:57.497657Z","shell.execute_reply.started":"2023-04-15T13:21:57.400790Z","shell.execute_reply":"2023-04-15T13:21:57.496548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(test_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:23:37.387652Z","iopub.execute_input":"2023-04-15T13:23:37.388337Z","iopub.status.idle":"2023-04-15T13:23:37.396621Z","shell.execute_reply.started":"2023-04-15T13:23:37.388301Z","shell.execute_reply":"2023-04-15T13:23:37.395454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({\"ImageId\": range(1, len(test_predictions) + 1), \"Label\": test_predictions}).to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-04-15T13:40:31.505645Z","iopub.execute_input":"2023-04-15T13:40:31.506006Z","iopub.status.idle":"2023-04-15T13:40:31.561001Z","shell.execute_reply.started":"2023-04-15T13:40:31.505975Z","shell.execute_reply":"2023-04-15T13:40:31.560117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}