{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # for plotting beautiful graphs\n\n# train test split from sklearn\nfrom sklearn.model_selection import train_test_split\n\n# Import Torch \nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms, models\n# from torch.utils.data import SubsetRandomSampler\nfrom torch.autograd import Variable\nfrom torch import nn, optim\nimport torch.nn.functional as F\n\n# What's in the current directory?\nimport os\nprint(os.listdir(\"../input\"))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:53.318914Z","iopub.execute_input":"2023-02-27T15:00:53.320387Z","iopub.status.idle":"2023-02-27T15:00:53.330099Z","shell.execute_reply.started":"2023-02-27T15:00:53.320323Z","shell.execute_reply":"2023-02-27T15:00:53.328907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\", dtype=np.float32)\nfinal_test = pd.read_csv(\"../input/digit-recognizer/test.csv\", dtype=np.float32)\nsample_sub = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\ntrain.label.head()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:53.332143Z","iopub.execute_input":"2023-02-27T15:00:53.333451Z","iopub.status.idle":"2023-02-27T15:00:57.311846Z","shell.execute_reply.started":"2023-02-27T15:00:53.333409Z","shell.execute_reply":"2023-02-27T15:00:57.310509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.313929Z","iopub.execute_input":"2023-02-27T15:00:57.314309Z","iopub.status.idle":"2023-02-27T15:00:57.359395Z","shell.execute_reply.started":"2023-02-27T15:00:57.314272Z","shell.execute_reply":"2023-02-27T15:00:57.357990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Seperate the features and labels\ntargets_np = train.label.values\nfeatures_np = train.loc[:, train.columns != 'label'].values/255\n\n# Split into training and test set\nfeatures_train, features_test, target_train, target_test = train_test_split(features_np, targets_np, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.362050Z","iopub.execute_input":"2023-02-27T15:00:57.362399Z","iopub.status.idle":"2023-02-27T15:00:57.854395Z","shell.execute_reply.started":"2023-02-27T15:00:57.362360Z","shell.execute_reply":"2023-02-27T15:00:57.853134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create feature and targets tensor for train set. As you remember we need variable to accumulate gradients. Therefore first we create tensor, then we will create variable\nfeaturesTrain = torch.from_numpy(features_train)\ntargetsTrain = torch.from_numpy(target_train).type(torch.LongTensor) # data type is long\n\n# create feature and targets tensor for test set.\nfeaturesTest = torch.from_numpy(features_test)\ntargetsTest = torch.from_numpy(target_test).type(torch.LongTensor) # data type is long","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.857362Z","iopub.execute_input":"2023-02-27T15:00:57.857775Z","iopub.status.idle":"2023-02-27T15:00:57.865886Z","shell.execute_reply.started":"2023-02-27T15:00:57.857735Z","shell.execute_reply":"2023-02-27T15:00:57.864410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set batch size\nbatch_size = 256\n\n# Pytorch train and test sets\ntrain = torch.utils.data.TensorDataset(featuresTrain,targetsTrain)\ntest = torch.utils.data.TensorDataset(featuresTest,targetsTest)\n\n# data loader\ntrain_loader = torch.utils.data.DataLoader(train, batch_size = batch_size, shuffle = True)\ntest_loader = torch.utils.data.DataLoader(test, batch_size = batch_size, shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.867886Z","iopub.execute_input":"2023-02-27T15:00:57.868642Z","iopub.status.idle":"2023-02-27T15:00:57.880790Z","shell.execute_reply.started":"2023-02-27T15:00:57.868567Z","shell.execute_reply":"2023-02-27T15:00:57.879492Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize one of the images in data set\ndef visualize_image(data, index, pred=False, val=0):\n    '''This funtion can be used to visualize the images'''\n    plt.imshow(data[index].reshape(28,28))\n    plt.axis(\"off\")\n    plt.title(\"Handwritten Digit Image\")\n    plt.show()\nvisualize_image(features_np, 2)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.882226Z","iopub.execute_input":"2023-02-27T15:00:57.882585Z","iopub.status.idle":"2023-02-27T15:00:57.985889Z","shell.execute_reply.started":"2023-02-27T15:00:57.882550Z","shell.execute_reply":"2023-02-27T15:00:57.984056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"featuresTrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:57.988351Z","iopub.execute_input":"2023-02-27T15:00:57.989350Z","iopub.status.idle":"2023-02-27T15:00:58.002144Z","shell.execute_reply.started":"2023-02-27T15:00:57.989269Z","shell.execute_reply":"2023-02-27T15:00:57.999965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Classifier(nn.Module):\n    def __init__(self):\n        super().__init__()\n        # 5 Hidden Layer Network\n        self.fc1 = nn.Linear(28*28, 512)\n        self.fc2 = nn.Linear(512, 256)\n        self.fc3 = nn.Linear(256, 128)\n        self.fc4 = nn.Linear(128, 64)\n        self.fc5 = nn.Linear(64, 10)\n        \n        # Dropout module with 0.2 probbability\n        self.dropout = nn.Dropout(p=0.2)\n        # Add softmax on output layer\n        self.log_softmax = F.log_softmax\n        \n    def forward(self, x):\n        x = self.dropout(F.relu(self.fc1(x)))\n        x = self.dropout(F.relu(self.fc2(x)))\n        x = self.dropout(F.relu(self.fc3(x)))\n        x = self.dropout(F.relu(self.fc4(x)))\n        \n        x = self.log_softmax(self.fc5(x), dim=1)\n        \n        return x","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:58.004711Z","iopub.execute_input":"2023-02-27T15:00:58.005412Z","iopub.status.idle":"2023-02-27T15:00:58.025928Z","shell.execute_reply.started":"2023-02-27T15:00:58.005341Z","shell.execute_reply":"2023-02-27T15:00:58.023564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate our model\nmodel = Classifier()\n# Define our loss function\ncriterion = nn.NLLLoss()\n# Define the optimier\noptimizer = optim.Adam(model.parameters(), lr=0.0015)\n\nepochs = 25\nsteps = 0\nprint_every = 50\ntrain_losses, test_losses = [], []\n\nfor e in range(epochs):\n    running_loss = 0\n    for images, labels in train_loader:\n        steps += 1\n        # Prevent accumulation of gradients\n        optimizer.zero_grad()\n        # Make predictions\n        log_ps = model(images)\n        loss = criterion(log_ps, labels)\n        #backprop\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        if steps % print_every == 0:\n            test_loss = 0\n            accuracy = 0\n\n            # Turn off gradients for validation\n            with torch.no_grad():\n                model.eval()\n                for images, labels in test_loader:\n                    log_ps = model(images)\n                    test_loss += criterion(log_ps, labels)\n                    ps = torch.exp(log_ps)\n                    # Get our top predictions\n                    top_p, top_class = ps.topk(1, dim=1)\n                    equals = top_class == labels.view(*top_class.shape)\n                    accuracy += torch.mean(equals.type(torch.FloatTensor))\n\n            model.train()\n\n            train_losses.append(running_loss/len(train_loader))\n            test_losses.append(test_loss/len(test_loader))\n\n            print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n                  \"Training Loss: {:.3f}.. \".format(train_losses[-1]),\n                  \"Test Loss: {:.3f}.. \".format(test_losses[-1]),\n                  \"Test Accuracy: {:.3f}\".format(accuracy/len(test_loader)))","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:00:58.028411Z","iopub.execute_input":"2023-02-27T15:00:58.029982Z","iopub.status.idle":"2023-02-27T15:01:58.942564Z","shell.execute_reply.started":"2023-02-27T15:00:58.029899Z","shell.execute_reply":"2023-02-27T15:01:58.941119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\n%config InlineBackend.figure_format = 'retina'\n\nplt.plot(train_losses, label='Training loss')\nplt.plot(test_losses, label='Validation loss')\nplt.legend(frameon=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:02:07.559727Z","iopub.execute_input":"2023-02-27T15:02:07.561059Z","iopub.status.idle":"2023-02-27T15:02:07.941553Z","shell.execute_reply.started":"2023-02-27T15:02:07.561002Z","shell.execute_reply":"2023-02-27T15:02:07.940143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def view_classify(img, ps):\n    ''' Function for viewing an image and it's predicted classes.\n    '''\n    ps = ps.data.numpy().squeeze()\n\n    fig, (ax1, ax2) = plt.subplots(figsize=(6,9), ncols=2)\n    ax1.imshow(img.resize_(1, 28, 28).numpy().squeeze())\n    ax1.axis('off')\n    ax2.barh(np.arange(10), ps)\n    ax2.set_aspect(0.1)\n    ax2.set_yticks(np.arange(10))\n    ax2.set_yticklabels(np.arange(10))\n    ax2.set_title('Class Probability')\n    ax2.set_xlim(0, 1.1)\n\n    plt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:02:43.826764Z","iopub.execute_input":"2023-02-27T15:02:43.828000Z","iopub.status.idle":"2023-02-27T15:02:43.837235Z","shell.execute_reply.started":"2023-02-27T15:02:43.827935Z","shell.execute_reply":"2023-02-27T15:02:43.835986Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\ndef make_prediction(data):\n    images, labels = next(iter(data))\n\n    img = images[42].view(1, 784)\n    # Turn off gradients to speed up this part\n    with torch.no_grad():\n        logps = model(img)\n\n    # Output of the network are log-probabilities, need to take exponential for probabilities\n    ps = torch.exp(logps)\n    view_classify(img.view(1, 28, 28), ps)\nmake_prediction(test_loader)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:03:12.894595Z","iopub.execute_input":"2023-02-27T15:03:12.895075Z","iopub.status.idle":"2023-02-27T15:03:13.342149Z","shell.execute_reply.started":"2023-02-27T15:03:12.895033Z","shell.execute_reply":"2023-02-27T15:03:13.340777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_test_np = final_test.values/255\ntest_tn = torch.from_numpy(final_test_np)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:03:23.090546Z","iopub.execute_input":"2023-02-27T15:03:23.091053Z","iopub.status.idle":"2023-02-27T15:03:23.126896Z","shell.execute_reply.started":"2023-02-27T15:03:23.090992Z","shell.execute_reply":"2023-02-27T15:03:23.125570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fake_labels = np.zeros(final_test_np.shape)\nfake_labels = torch.from_numpy(fake_labels)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:03:33.865647Z","iopub.execute_input":"2023-02-27T15:03:33.866095Z","iopub.status.idle":"2023-02-27T15:03:33.872361Z","shell.execute_reply.started":"2023-02-27T15:03:33.866051Z","shell.execute_reply":"2023-02-27T15:03:33.870953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_tn_data = torch.utils.data.TensorDataset(test_tn, fake_labels)\n\nsubmission_loader = torch.utils.data.DataLoader(submission_tn_data, batch_size = batch_size, shuffle = False)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:04:07.741056Z","iopub.execute_input":"2023-02-27T15:04:07.741461Z","iopub.status.idle":"2023-02-27T15:04:07.748050Z","shell.execute_reply.started":"2023-02-27T15:04:07.741418Z","shell.execute_reply":"2023-02-27T15:04:07.746399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"make_prediction(submission_loader)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:04:09.454106Z","iopub.execute_input":"2023-02-27T15:04:09.454818Z","iopub.status.idle":"2023-02-27T15:04:09.908639Z","shell.execute_reply.started":"2023-02-27T15:04:09.454773Z","shell.execute_reply":"2023-02-27T15:04:09.907219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making it submission ready\nsubmission = [['ImageId', 'Label']]\n\n# Turn off gradients for validation\nwith torch.no_grad():\n    model.eval()\n    image_id = 1\n    for images, _ in submission_loader:\n        log_ps = model(images)\n        ps = torch.exp(log_ps)\n        top_p, top_class = ps.topk(1, dim=1)\n        \n        for prediction in top_class:\n            submission.append([image_id, prediction.item()])\n            image_id += 1","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:04:25.179277Z","iopub.execute_input":"2023-02-27T15:04:25.179721Z","iopub.status.idle":"2023-02-27T15:04:26.293729Z","shell.execute_reply.started":"2023-02-27T15:04:25.179673Z","shell.execute_reply":"2023-02-27T15:04:26.292704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(submission)\nsubmission_df.columns = submission_df.iloc[0]\nsubmission_df = submission_df.drop(0, axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:04:42.257688Z","iopub.execute_input":"2023-02-27T15:04:42.258560Z","iopub.status.idle":"2023-02-27T15:04:42.273073Z","shell.execute_reply.started":"2023-02-27T15:04:42.258506Z","shell.execute_reply":"2023-02-27T15:04:42.271524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-02-27T15:04:43.150638Z","iopub.execute_input":"2023-02-27T15:04:43.151759Z","iopub.status.idle":"2023-02-27T15:04:43.191541Z","shell.execute_reply.started":"2023-02-27T15:04:43.151711Z","shell.execute_reply":"2023-02-27T15:04:43.190605Z"},"trusted":true},"execution_count":null,"outputs":[]}]}