{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split as tts\nimport matplotlib.pyplot as plt\n\nprint(\"Setup complete\")","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:22.485254Z","iopub.execute_input":"2021-08-12T13:12:22.485912Z","iopub.status.idle":"2021-08-12T13:12:25.045062Z","shell.execute_reply.started":"2021-08-12T13:12:22.485788Z","shell.execute_reply":"2021-08-12T13:12:25.043833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\", dtype=np.float32)\nfinal_test = pd.read_csv(\"../input/digit-recognizer/test.csv\", dtype=np.float32)\nsample_sub = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:25.046741Z","iopub.execute_input":"2021-08-12T13:12:25.047036Z","iopub.status.idle":"2021-08-12T13:12:31.964156Z","shell.execute_reply.started":"2021-08-12T13:12:25.047008Z","shell.execute_reply":"2021-08-12T13:12:31.963248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.label.head()\ntrain.info()","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:31.966367Z","iopub.execute_input":"2021-08-12T13:12:31.966745Z","iopub.status.idle":"2021-08-12T13:12:32.027978Z","shell.execute_reply.started":"2021-08-12T13:12:31.966708Z","shell.execute_reply":"2021-08-12T13:12:32.026804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets_np = train.label.values\nfeatures_np = train.loc[:, train.columns != 'label'].values/255\n\n# Split into training and test set\nfeatures_train, features_test, target_train, target_test = tts(features_np, targets_np, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.030197Z","iopub.execute_input":"2021-08-12T13:12:32.030657Z","iopub.status.idle":"2021-08-12T13:12:32.732116Z","shell.execute_reply.started":"2021-08-12T13:12:32.030613Z","shell.execute_reply":"2021-08-12T13:12:32.730855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"featuresTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(target_train).type(torch.LongTensor) # data type is long\n\n# create feature and targets tensor for test set.\nfeaturesTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(target_test).type(torch.LongTensor) # data type is long","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.733565Z","iopub.execute_input":"2021-08-12T13:12:32.733883Z","iopub.status.idle":"2021-08-12T13:12:32.763121Z","shell.execute_reply.started":"2021-08-12T13:12:32.733855Z","shell.execute_reply":"2021-08-12T13:12:32.762294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.5,), (0.5,))\n])\n\ntrain = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)\ntest = torch.utils.data.TensorDataset(featuresTest,targetsTest)\n\nbatch_size = 32\n# data loader\ntrainloader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = True)\ntestloader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.764362Z","iopub.execute_input":"2021-08-12T13:12:32.764803Z","iopub.status.idle":"2021-08-12T13:12:32.772845Z","shell.execute_reply.started":"2021-08-12T13:12:32.764771Z","shell.execute_reply":"2021-08-12T13:12:32.771967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = nn.Sequential(\n    nn.Linear(784,128),\n    nn.ReLU(),\n    nn.Linear(128, 64),\n    nn.ReLU(),\n    nn.Linear(64,10),\n    # nn.Sigmoid()\n)\n\ncriterion = nn.CrossEntropyLoss()\n\nimages, labels = next(iter(trainloader))\n\nimages = images.view(images.shape[0],-1)\n\nlogits = model(images)\n\nloss = criterion(logits, labels)\n\nprint(loss)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.774107Z","iopub.execute_input":"2021-08-12T13:12:32.774588Z","iopub.status.idle":"2021-08-12T13:12:32.916431Z","shell.execute_reply.started":"2021-08-12T13:12:32.774554Z","shell.execute_reply":"2021-08-12T13:12:32.915259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Implementing a sample NN","metadata":{}},{"cell_type":"code","source":"model = nn.Sequential(\n    nn.Linear(784,128),\n    nn.ReLU(),\n    nn.Linear(128, 64),\n    nn.ReLU(),\n    nn.Linear(64,10),\n    nn.LogSoftmax(dim=1)\n)\n\ncriterion = nn.NLLLoss()\nimages, labels = next(iter(trainloader))\nimages = images.view(images.shape[0],-1)\n\nlogits = model(images)\n\nloss = criterion(logits, labels)\n\nprint(loss)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.919743Z","iopub.execute_input":"2021-08-12T13:12:32.920302Z","iopub.status.idle":"2021-08-12T13:12:32.939090Z","shell.execute_reply.started":"2021-08-12T13:12:32.920260Z","shell.execute_reply":"2021-08-12T13:12:32.937873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using Autograd","metadata":{}},{"cell_type":"code","source":"model = nn.Sequential(\n    nn.Linear(784,128),\n    nn.ReLU(),\n    nn.Linear(128, 64),\n    nn.ReLU(),\n    nn.Linear(64,10),\n    nn.LogSoftmax(dim=1)\n)\n\ncriterion = nn.NLLLoss()\nimages, labels = next(iter(trainloader))\nimages = images.view(images.shape[0],-1)\n\nlogits = model(images)\n\nloss = criterion(logits, labels)\n\nprint(\"Before backward pass:\", model[0].weight.grad)\n\nloss.backward()\n\nprint(\"After backward pass:\", model[0].weight.grad)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.941095Z","iopub.execute_input":"2021-08-12T13:12:32.941968Z","iopub.status.idle":"2021-08-12T13:12:32.969154Z","shell.execute_reply.started":"2021-08-12T13:12:32.941924Z","shell.execute_reply":"2021-08-12T13:12:32.968174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Optimizing the NN using optim","metadata":{}},{"cell_type":"code","source":"from torch import optim\n\noptimizer = optim.SGD(model.parameters(), lr=0.01)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.970550Z","iopub.execute_input":"2021-08-12T13:12:32.971171Z","iopub.status.idle":"2021-08-12T13:12:32.977123Z","shell.execute_reply.started":"2021-08-12T13:12:32.971131Z","shell.execute_reply":"2021-08-12T13:12:32.975901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Initial weights:\", model[0].weight)\n\nimages, labels = next(iter(trainloader))\nimages.resize_(784, 64)\n\noptimizer.zero_grad()","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.978635Z","iopub.execute_input":"2021-08-12T13:12:32.979138Z","iopub.status.idle":"2021-08-12T13:12:32.997068Z","shell.execute_reply.started":"2021-08-12T13:12:32.979093Z","shell.execute_reply":"2021-08-12T13:12:32.996245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Summing up together","metadata":{}},{"cell_type":"code","source":"import torch\nfrom torch import nn\nimport torch.nn.functional as F\nfrom torchvision import datasets, transforms\nfrom torch import optim\n\nprint(\"Setup complete\")\n\nmodel = nn.Sequential(\n    nn.Linear(784, 128),\n    nn.ReLU(),\n    nn.Linear(128, 64),\n    nn.ReLU(),\n    nn.Linear(64, 10),\n    nn.LogSoftmax(dim=1)\n)\n\ncriterion = nn.NLLLoss()\n\noptimizer = optim.SGD(model.parameters(), lr=0.01)\n\nepochs = 50\n\nfor e in range(epochs):\n    running_loss = 0\n\n    for images, labels in trainloader:\n        images = images.view(images.shape[0], -1)\n\n        optimizer.zero_grad()\n\n        output = model(images)\n        loss = criterion(output, labels)\n\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n\n    else:\n        print(\"Running loss=\", running_loss/len(trainloader))","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:12:32.998521Z","iopub.execute_input":"2021-08-12T13:12:32.998978Z","iopub.status.idle":"2021-08-12T13:14:07.119457Z","shell.execute_reply.started":"2021-08-12T13:12:32.998945Z","shell.execute_reply":"2021-08-12T13:14:07.118118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Checking predictions","metadata":{}},{"cell_type":"code","source":"%matplotlib inline\nimport helper\n\nimages, labels = next(iter(trainloader))\n\nimg = images[0].view(1, 784)\n# Turn off gradients to speed up this part\nwith torch.no_grad():\n    logps = model(img)\n\n# Output of the network are log-probabilities, need to take exponential for probabilities\nps = torch.exp(logps)\n# helper.view_classify(img.view(1, 28, 28), ps)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.121154Z","iopub.execute_input":"2021-08-12T13:14:07.121533Z","iopub.status.idle":"2021-08-12T13:14:07.145962Z","shell.execute_reply.started":"2021-08-12T13:14:07.121500Z","shell.execute_reply":"2021-08-12T13:14:07.144893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(max(ps.tolist()))","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.147541Z","iopub.execute_input":"2021-08-12T13:14:07.147883Z","iopub.status.idle":"2021-08-12T13:14:07.155733Z","shell.execute_reply.started":"2021-08-12T13:14:07.147853Z","shell.execute_reply":"2021-08-12T13:14:07.154493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_classify(img, ps):\n    ''' Function for viewing an image and it's predicted classes.\n    '''\n    ps = ps.data.numpy().squeeze()\n\n    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())\n    ax1.axis('off')\n    ax2.barh(np.arange(10), ps)\n    ax2.set_aspect(0.1)\n    ax2.set_yticks(np.arange(10))\n    ax2.set_yticklabels(np.arange(10))\n    ax2.set_title('Class Probability')\n    ax2.set_xlim(0, 1.1)\n\n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.157332Z","iopub.execute_input":"2021-08-12T13:14:07.157629Z","iopub.status.idle":"2021-08-12T13:14:07.166536Z","shell.execute_reply.started":"2021-08-12T13:14:07.157600Z","shell.execute_reply":"2021-08-12T13:14:07.165547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\ndef make_prediction(data):\n    images, labels = next(iter(data))\n\n    img = images[22].view(1, 784)\n    # Turn off gradients to speed up this part\n    with torch.no_grad():\n        logps = model(img)\n\n    # Output of the network are log-probabilities, need to take exponential for probabilities\n    ps = torch.exp(logps)\n    view_classify(img.view(1, 28, 28), ps)\nmake_prediction(testloader)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.168036Z","iopub.execute_input":"2021-08-12T13:14:07.168430Z","iopub.status.idle":"2021-08-12T13:14:07.495413Z","shell.execute_reply.started":"2021-08-12T13:14:07.168397Z","shell.execute_reply":"2021-08-12T13:14:07.494655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_np = final_test.values/255\ntest_tn = torch.from_numpy(final_test_np)\n\nfake_labels = np.zeros(final_test_np.shape)\nfake_labels = torch.from_numpy(fake_labels)\n\nsubmission_tn_data = torch.utils.data.TensorDataset(test_tn, fake_labels)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_tn_data, batch_size = batch_size, shuffle = False)\n\nmake_prediction(submission_loader)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.496471Z","iopub.execute_input":"2021-08-12T13:14:07.496885Z","iopub.status.idle":"2021-08-12T13:14:07.841056Z","shell.execute_reply.started":"2021-08-12T13:14:07.496847Z","shell.execute_reply":"2021-08-12T13:14:07.840270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = [['ImageId', 'Label']]\n\n# Turn off gradients for validation\nwith torch.no_grad():\n    model.eval()\n    image_id = 1\n    for images, _ in submission_loader:\n        log_ps = model(images)\n        ps = torch.exp(log_ps)\n        top_p, top_class = ps.topk(1, dim=1)\n        \n        for prediction in top_class:\n            submission.append([image_id, prediction.item()])\n            image_id += 1","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:07.842462Z","iopub.execute_input":"2021-08-12T13:14:07.842761Z","iopub.status.idle":"2021-08-12T13:14:08.821021Z","shell.execute_reply.started":"2021-08-12T13:14:07.842726Z","shell.execute_reply":"2021-08-12T13:14:08.819842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(submission)\nsubmission_df.columns = submission_df.iloc[0]\nsubmission_df = submission_df.drop(0, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:08.822826Z","iopub.execute_input":"2021-08-12T13:14:08.823320Z","iopub.status.idle":"2021-08-12T13:14:08.839610Z","shell.execute_reply.started":"2021-08-12T13:14:08.823275Z","shell.execute_reply":"2021-08-12T13:14:08.838543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-12T13:14:08.841131Z","iopub.execute_input":"2021-08-12T13:14:08.841752Z","iopub.status.idle":"2021-08-12T13:14:08.915620Z","shell.execute_reply.started":"2021-08-12T13:14:08.841708Z","shell.execute_reply":"2021-08-12T13:14:08.914385Z"},"trusted":true},"execution_count":null,"outputs":[]}]}