{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # data visualization\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport torch # pytorch library\nfrom torch.utils.data import TensorDataset, DataLoader\n\n# Read Train Data\ntrain_data = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndisplay(train_data.head())\n\n# Extract labels\ny = train_data.label.values\n\n# Extract images\nX = train_data.drop([\"label\"], axis=1).values.reshape((-1, 1, 28, 28)) / 255.0\nprint(X.shape)\n\n# Extract validation set\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=2334)\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)\n\n# Create tensors\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\nX_test_tensor = torch.tensor(X_test, dtype=torch.float32)\ny_train_tensor = torch.tensor(y_train, dtype=torch.int64)\ny_test_tensor = torch.tensor(y_test, dtype=torch.int64)\n\n# Create a torch datasets\ntrain_dataset = TensorDataset(X_train_tensor, y_train_tensor)\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True, num_workers=4)\n\ntest_dataset = TensorDataset(X_test_tensor, y_test_tensor)\ntest_loader = DataLoader(test_dataset, batch_size=128, shuffle=True, num_workers=4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nimage, label = next(iter(train_loader))\nplt.imshow(image[0,0,:]);\nprint(label[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"image, label = next(iter(test_loader))\nplt.imshow(image[0,0,:]);\nprint(label[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\n\n# Define CNN\nclass Net(nn.Module):\n    def __init__(self):\n        super().__init__()\n        \n        self.drop = nn.Dropout(0.5)\n        \n        # (B, 1, 28, 28) -> (B, 32, 12, 12)\n        self.conv1 = nn.Conv2d(1, 32, 5)\n        self.batch1 = nn.BatchNorm2d(32)\n        self.pool = nn.MaxPool2d(2, 2)\n        \n        # (B, 32, 12, 12) -> (B, 64, 4, 4)\n        self.conv2 = nn.Conv2d(32, 64, 3)\n        self.batch2 = nn.BatchNorm2d(64)\n        self.conv3 = nn.Conv2d(64, 64, 3)\n        self.batch3 = nn.BatchNorm2d(64)    \n            \n        self.fc1 = nn.Linear(64 * 4 * 4, 64)\n        self.fc2 = nn.Linear(64, 10)\n\n    def forward(self, x):\n        x = F.relu(self.batch1(self.conv1(x)))\n        x = self.pool(x)\n        x = self.drop(x)\n        \n        x = F.relu(self.batch2(self.conv2(x)))\n        x = F.relu(self.batch3(self.conv3(x)))\n        x = self.pool(x)\n        x = self.drop(x)\n        \n        x = x.view(-1, 64 * 4 * 4)\n        x = F.relu(self.fc1(x))\n        x = self.drop(x)\n        x = self.fc2(x)\n        return F.log_softmax(x, dim=1)\n\nnet = Net()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.optim as optim\n\ncriterion = nn.NLLLoss()\noptimizer = optim.Adam(net.parameters(), lr=0.001)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if torch.cuda.is_available():  \n  dev = \"cuda:0\" \nelse:  \n  dev = \"cpu\"\n\ndevice = torch.device(dev)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Implement a function for the validation pass\ndef validation(model, testloader, criterion):\n    test_loss = 0\n    accuracy = 0\n    for images, labels in testloader:\n        images, labels = images.to(device), labels.to(device)\n        \n        output = model.forward(images)\n        test_loss += criterion(output, labels).item()\n\n        ps = torch.exp(output)\n        equality = (labels.data == ps.max(dim=1)[1])\n        accuracy += equality.type(torch.FloatTensor).mean()\n    \n    return test_loss, accuracy","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 48\nsteps = 0\nrunning_loss = 0\nprint_every = 40\n\nnet.to(device)\n\nfor e in range(epochs):\n    net.train()\n    for images, labels in train_loader:\n        steps += 1\n        \n        images, labels = images.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        \n        output = net.forward(images)\n        loss = criterion(output, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n        \n        if steps % print_every == 0:\n            # Make sure network is in eval mode for inference\n            net.eval()\n            \n            # Turn off gradients for validation, saves memory and computations\n            with torch.no_grad():\n                test_loss, accuracy = validation(net, test_loader, criterion)\n                \n            print(\"Epoch: {}/{}.. \".format(e+1, epochs),\n                  \"Training Loss: {:.3f}.. \".format(running_loss/print_every),\n                  \"Test Loss: {:.3f}.. \".format(test_loss/len(test_loader)),\n                  \"Test Accuracy: {:.3f}\".format(accuracy/len(test_loader)))\n            \n            running_loss = 0\n            \n            # Make sure training is back on\n            net.train()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read Train Data\noutput_data = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\ndisplay(output_data.head())\n\n# Extract images\noutput_images = output_data.values.reshape((-1, 1, 28, 28)) / 255.0\nprint(output_images.shape)\n\n# Create tensor\noutput_image_tensor = torch.tensor(output_images, dtype=torch.float32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net.eval()\n\nresult = np.zeros(output_images.shape[0], dtype=np.int64)\n\nnet.to(torch.device(\"cpu\"))\n\nwith torch.no_grad():\n    for i in range(output_images.shape[0]):\n        image = output_image_tensor[i, 0, :, :].view(1, 1, 28, 28)\n                \n        output = net.forward(image)\n        ps = torch.exp(output)\n        label = ps.max(dim=1)[1][0].cpu().numpy()\n        result[i] = label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(result[:5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\nsample_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame({\"ImageId\": range(1, len(result)+1), \"Label\": result})\ndisplay(submission.head())\nsubmission.to_csv('my_submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}