{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importin libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm import tqdm, tqdm_notebook","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training on GPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading images"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_images=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_target=train_images['label']\ntrain_images=train_images.drop(['label'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Normolizing and reshaping data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_images=train_images/255.0\ntest_images=test_images/255.0\n\ntrain_images=train_images.values.reshape(-1, 1, 28,28)\ntest_images=test_images.values.reshape(-1,1, 28,28)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Class to create dataset "},{"metadata":{"trusted":true},"cell_type":"code","source":"class MNISTDataset(Dataset):\n  def __init__(self, x_train, y_train, mode):\n    self.x_train=x_train\n    self.y_train=y_train\n    self.mode=mode\n  def __len__(self, ):\n    return len(self.x_train)\n  def __getitem__(self, index):\n    if self.mode=='test':\n      return self.x_train[index]\n    else:\n      return self.x_train[index], self.y_train[index]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN class"},{"metadata":{"trusted":true},"cell_type":"code","source":"class DigitRecognize(nn.Module):\n  def __init__(self, n_classes):\n    super().__init__()\n    self.conv1=nn.Sequential(\n        nn.Conv2d(in_channels=1, out_channels=16, kernel_size=5, padding=2, stride=1),\n        nn.ReLU(),\n    )\n    self.conv2=nn.Sequential(\n        nn.Conv2d(in_channels=16, out_channels=32, kernel_size=5, padding=2, stride=1),\n        nn.ReLU(),\n        nn.MaxPool2d(2)\n    )\n    self.conv3=nn.Sequential(\n        nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=2, stride=1),\n        nn.ReLU()\n    )\n    self.conv4=nn.Sequential(\n        nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=2, stride=1),\n        nn.ReLU(),\n        nn.MaxPool2d(2)\n    )\n    self.conv5=nn.Sequential(\n        nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, padding=2, stride=1),\n        nn.ReLU(),\n        nn.MaxPool2d(2)\n    )\n    self.dropOut=nn.Dropout2d(0.33)\n    self.fc1=nn.Linear(1600, 1000)\n    self.dropOut_2=nn.Dropout()\n    self.fc2=nn.Linear(1000, 10)\n  def forward(self, x):\n    x=torch.tensor(x, dtype=torch.float32)\n    x=self.conv1(x)\n    x=self.conv2(x)\n    x=self.conv3(x)\n    x=self.dropOut(x)\n    x=self.conv4(x)\n    x=self.conv5(x)\n    x=self.dropOut(x)\n    x=x.view(x.size(0),-1)\n    x=self.fc1(x)\n    x=self.dropOut_2(x)\n    x=self.fc2(x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training one epoch"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_epoch(model, train_data, optimizer, criterion):\n  running_loss=0.0\n  running_correct=0\n  running_total=0\n  model.train()\n  for x_batch, y_batch in train_data:\n    y_batch=y_batch.type(torch.LongTensor)\n    x_batch=x_batch.to(DEVICE)\n    y_batch=y_batch.to(DEVICE)\n    output=model(x_batch)\n    optimizer.zero_grad()\n    loss=criterion(output, y_batch)\n    preds=torch.argmax(output, 1)\n    loss.backward()\n    optimizer.step()\n    running_loss+=loss*x_batch.size(0)\n    running_correct+=(preds==y_batch).sum().item()\n    running_total+=x_batch.size(0)\n  train_loss=running_loss/running_total\n  train_acc=running_correct/running_total\n  return train_loss, train_acc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Function to get loss and accuracy on validation data"},{"metadata":{"trusted":true},"cell_type":"code","source":"def val_epoch(model, test_data, criterion):\n  running_loss=0.0\n  running_correct=0\n  running_total=0\n  with torch.no_grad():\n    for x_batch, y_batch in test_data:\n      y_batch=y_batch.type(torch.LongTensor)\n      x_batch=x_batch.to(DEVICE)\n      y_batch=y_batch.to(DEVICE)\n      model.eval()\n      output=model(x_batch)\n      loss=criterion(output, y_batch)\n      preds=torch.argmax(output, 1)\n      running_loss+=loss*x_batch.size(0)\n      running_correct+=(preds==y_batch).sum().item()\n      running_total+=x_batch.size(0)\n  test_loss=running_loss/running_total\n  test_acc=running_correct/running_total\n  return test_loss, test_acc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(model,train_dataset, batch_size, epochs):\n  max_acc=0\n  history=[]\n  with tqdm(desc=\"epochs\", total=epochs) as tbar:\n      for i in range(epochs):\n        if i%10==0:\n          train_data, test_data=train_test_split(train_dataset, test_size=0.1)\n          test_loader=DataLoader(test_data, batch_size=batch_size, shuffle=False)\n          train_loader=DataLoader(train_data, batch_size=batch_size, shuffle=True)\n            #Shuffle data every 10 epoch\n        criterion=nn.CrossEntropyLoss()\n        optimizer=torch.optim.Adam(model.parameters(), lr=0.001)\n        train_loss, train_acc=train_epoch(model, train_loader, optimizer, criterion)\n        test_loss, test_acc=val_epoch(model, test_loader, criterion)\n        print()\n        if test_acc>max_acc:\n          max_acc=test_acc\n          torch.save(model.state_dict(), \"MNISTModel\")\n          print(f'Model with accuracy: {test_acc} is saved')\n        print(\"Loss:\", test_loss.item())\n        print(\"Accuracy:\",test_acc)\n        history.append((train_loss, train_acc,  test_loss, test_acc))\n        tbar.update(1)\n        print(f'Epoch {i+1} train_loss: {train_loss:.3}  val_loss: {test_loss:.3} train_acc: {train_acc:.5} val_acc: {test_acc:.5}')\n  return history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model instantiation "},{"metadata":{"trusted":true},"cell_type":"code","source":" model=DigitRecognize(10).to(DEVICE)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Create datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset=MNISTDataset(train_images, train_target, 'train')\ntest_dataset=MNISTDataset(test_images,None , 'test')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Start training"},{"metadata":{"trusted":true},"cell_type":"code","source":"history=train(model,train_dataset, 128, 100) #You can try to change count of epochs and batch size","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Result of training"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_loss, train_acc, test_loss, test_acc=zip(*history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization\nUse matplotlib to plot graphs of loss and accuracy"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,9))\nplt.plot(train_loss, label=\"Train loss\")\nplt.plot(test_loss, label=\"Test loss\")\nplt.legend()\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,9))\nplt.plot(train_acc, label=\"Train accuracy\")\nplt.plot(test_acc, label=\"Test accuracy\")\nplt.legend()\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Accuracy\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prediction function"},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict(model, test_data,batch_size):\n  logits=[]\n  test_loader=DataLoader(test_data, batch_size=batch_size, shuffle=False)\n  with torch.no_grad():\n    model.eval()\n    for x_batch in test_loader:\n      x_batch=x_batch.to(DEVICE)\n      outputs=model(x_batch).cpu()\n      preds=torch.argmax(outputs, 1)\n      for i in preds:\n        logits.append(i.item())\n  return logits","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"load_model=DigitRecognize(10).to(DEVICE)\nload_model.load_state_dict(torch.load(\"MNISTModel\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_res=predict(load_model,test_dataset,64)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preparing data to submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"submit = pd.DataFrame(columns=['ImageId'])\nsubmit['ImageId'] = range(1,28001)\nsubmit['Label']=predict_res\nsubmit.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submit.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}