{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport sys\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\nfrom torch.utils.data import TensorDataset, DataLoader, Dataset\nfrom sklearn.model_selection import train_test_split\nfrom torch import nn, optim\nimport torch.nn.functional as F\nfrom skimage.transform import rotate\n\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Using GPU\nprint(sys.version)\ndevice = 'cuda'\n#Checking for GPU\nprint(torch.version.cuda)\nprint(torch.cuda.device_count())\nprint(torch.cuda.is_available())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntransform1 = transforms.Compose([transforms.ToPILImage(), transforms.RandomAffine(degrees = 30, translate = (0.1, 0.1), scale = (0.98, 1.02)), transforms.ToTensor()])\ntransform2 = transforms.Compose([transforms.ToPILImage(), transforms.RandomAffine(degrees = 15, translate = (0.15, 0.15), scale = (0.95, 1.05)), transforms.ToTensor()])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/digit-recognizer/train.csv')\ntrain = torch.from_numpy(df.values)\nx_train = (train[:, 1:])#.to(device)\ny_train = (train[:, 0])#.to(device)\ndel train, df\n\n\ndf = pd.read_csv(\"../input/digit-recognizer/test.csv\")\nx_test = torch.from_numpy(df.values).to(device)\nprint(x_test.shape, y_train.shape, x_train.shape, y_train.max())\n\nn, c = x_train.shape #n=no. of training egs & c=n_H *n_W for each eg\nprint(n, c)\nprint(x_train.shape, y_train.shape)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_cv, y_train, y_cv = train_test_split(x_train, y_train, test_size = 0.1, random_state = random_seed)\nn_train, _ = x_train.shape\nn_cv, _ = x_cv.shape\nprint(n_train, n_cv)\nx_train#.to(device)\ny_train#.to(device)\nx_cv#.to(device)\ny_cv#.to(device)\nprint(x_train.shape, y_train.shape, x_cv.shape, y_cv.shape, x_train.dtype, y_train.dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''x_train_temp = x_train.cpu().numpy().astype(np.float32)\nprint(x_train_temp.shape, x_train_temp.dtype)\n\nfinal_train_x = []\nfinal_train_y = []\nfor i in range(x_train.shape[0]):\n    final_train_x.append(x_train_temp[i])\n    final_train_x.append(rotate(x_train_temp[i].reshape((28, 28)), angle=15, mode = 'wrap').reshape((784)))\n    final_train_x.append(rotate(x_train_temp[i].reshape((28, 28)), angle=8, mode = 'wrap').reshape((784)))\n    final_train_x.append(rotate(x_train_temp[i].reshape((28, 28)), angle=-15, mode = 'wrap').reshape((784)))\n    final_train_x.append(rotate(x_train_temp[i].reshape((28, 28)), angle=-8, mode = 'wrap').reshape((784)))\n    for j in range(5):\n        final_train_y.append(y_train[i].cpu().numpy().reshape(1))'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#final_train_x = np.reshape(final_train_x, (len(final_train_x.shape), 28*28))\n#final_train_y = np.reshape(final_train_y, (-1))\n\n'''print(final_train_x[0].dtype, final_train_y[0].dtype)\n\nrandom_seed = 1\nx_train = torch.from_numpy(np.array(final_train_x)).to(device)\ny_train = torch.from_numpy(np.array(final_train_y)).squeeze().to(device)\n\nprint(x_train.shape, y_train.shape, x_cv.shape, y_cv.shape, x_train.dtype, y_train.dtype)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''transformed_images =[]\ntransformed_labels = []\nx_train_trans = 0\ny_train_trans = 0\n\n\nfor i in range(n_train):\n    orig = x_train[i].cpu().numpy().astype(np.uint8).reshape((28, 28))\n    transformed1 = transform1(orig)\n    transformed2 = transform2(orig)\n    transformed_images.append(transformed1)\n    transformed_images.append(transformed2)\n    transformed_labels.append(y_train[i].reshape((-1, 1)))\n    transformed_labels.append(y_train[i].reshape((-1, 1)))\n    \nprint(transformed_images[1].shape, transformed_labels[1].shape)\n\n\nx_train_trans = torch.cat(transformed_images, dim=0)#.to(device)\ny_train_trans = torch.cat(transformed_labels, dim=0)#.to(device)\nprint(x_train_trans.shape, y_train_trans.shape)\n\n\nx_train = torch.cat((x_train_trans, x_train), dim=0)#.to(device)\ny_train = torch.cat((y_train_trans, y_train.reshape((-1, 1))), dim=0)#.to(device)\ny_train = y_train.squeeze()\nprint(x_train.shape, y_train.shape)\n\n'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''index = 3\ntester = x_train[index].cpu().numpy().astype(np.uint8).reshape((28, 28))\ntester3 = transformed_images[index].cpu().numpy().astype(np.uint8).reshape((28,28))\n\n\nprint (y_train[index])\ntester1 = transform1(tester)\ntester2 = transform2(tester)\n\nplt.imshow(tester2.reshape((28, 28)), cmap='gray')'''\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''tester = x_test[7].cpu().numpy().astype(np.uint8).reshape((28, 28, 1))\n\nplt.imshow(tester.reshape((28, 28)), cmap='gray')\n\nprint (tester.shape)'''\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index = 5\nplt.imshow(x_train[index].cpu().reshape((28, 28)), cmap='gray')\nprint (y_train[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index = 5\nplt.imshow(x_cv[index].cpu().reshape((28, 28)), cmap='gray')\nprint (y_cv[index])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain_ds = TensorDataset(x_train, y_train)\nvalid_ds = TensorDataset(x_cv, y_cv)\n\ndef get_data(train_ds, bs):\n    return DataLoader(train_ds, batch_size =bs, shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(x, y):\n  x = x/255.0\n  return x.reshape(-1, 1, 28, 28).to(device), y.to(device)\n\n\nclass WrappedDataLoader:\n    def __init__(self, dl, func):\n        self.dl = dl\n        self.func = func\n\n    def __len__(self):\n        return len(self.dl)\n\n    def __iter__(self):\n        batches = iter(self.dl)\n        for b in batches:\n            yield (self.func(*b))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nclass Mnist_CNN(nn.Module):\n  def __init__(self):\n    super().__init__()\n    self.conv1 = nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2) #n_C_prev=1(no RGB Grayscale) n_C=16(#filters)\n    self.conv2 = nn.Conv2d(32, 48, kernel_size=5, stride=1, padding=2) #n_C_prev=16 n_C=16\n    self.conv3 = nn.Conv2d(48, 64, kernel_size=3, stride=1, padding=1) #n_C_prev=16 n_C=10\n    self.conv4 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1) #n_C_prev=16 n_C=10\n    self.lin1 = nn.Linear(3136, 256)\n    self.lin2 = nn.Linear(256, 10)\n    self.norm1 = nn.BatchNorm2d(32)\n    self.norm2 = nn.BatchNorm2d(48)\n    self.norm3 = nn.BatchNorm2d(64)\n    self.drop1 = nn.Dropout2d(p=0.4)\n    self.drop2 = nn.Dropout2d(p=0.4)\n    self.norm4 = nn.BatchNorm1d(256)\n    self.drop3 = nn.Dropout(p=0.4)\n    #self.lin2 = nn.Linear(128, 10)\n    \n\n\n  def forward(self, xb):\n    xb = xb.view(-1, 1, 28, 28)\n    #1\n    xb = F.relu(self.conv1(xb))\n    xb = self.norm1(xb)\n    #xb = F.avg_pool2d(xb, 2, 2)\n    \n    #2\n    xb = F.relu(self.conv2(xb))\n    xb = self.norm2(xb)\n    xb = F.avg_pool2d(xb, 2, 2)\n    xb = self.drop2(xb)\n    \n    #3\n    xb = F.relu(self.conv3(xb))\n    xb = self.norm3(xb)\n    \n    #4\n    xb = F.relu(self.conv4(xb))\n    xb = self.norm3(xb)\n    xb = F.avg_pool2d(xb, 2, 2)\n    xb = self.drop2(xb)\n    \n    #fc1\n    xb = torch.flatten(xb, 1, 3)\n    xb = F.relu(self.lin1(xb))\n    xb = self.norm4(xb)\n    xb = self.drop3(xb)\n    \n    #fc2\n    xb = self.lin2(xb)\n    return xb.view(-1, xb.size(1))\n\nloss_func = F.cross_entropy \nmodel = Mnist_CNN()\nmodel.float()\nmodel.to(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(out, yb):\n    pred = torch.argmax(out, keepdim= False, dim=1)\n    return (pred == yb).float().mean() \n'''for using negative log likelihood loss and log softmax activation,\nPytorch provides a single function F.cross_entropy that combines the two'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fit(model, epochs, train_dl, valid_dl, opt):\n    losses = []\n    \n    for epoch in range(epochs):\n        train_loss =0\n        train_acc =0\n        model.train()\n        for xb, yb in train_dl:\n            xb.to(device)\n            yb.to(device)\n            pred = model(xb.float())\n            pred.to(device)\n            train_acc += accuracy(pred, yb)\n            loss = loss_func(pred, yb)\n            train_loss += loss\n            #back propogation\n            loss.backward()\n            opt.step()\n            opt.zero_grad()      \n\n        losses.append(train_loss)\n        print(\"Iteration no: \"+ str(epoch), \"loss = \"+str(losses[epoch].item()))\n        print(\"Accuracy of train set:\", train_acc/len(train_dl))\n        lr_scheduler.step(train_loss/len(train_dl))\n        \n        \n        model.eval()    \n        with torch.no_grad():\n            valid_acc=0\n            loss_valid = 0\n            for xb_valid, yb_valid in valid_dl:\n                xb_valid.to(device)\n                yb_valid.to(device)\n                pred_valid = model(xb_valid.float())\n                pred_valid.to(device)\n                valid_acc += accuracy(pred_valid, yb_valid)\n                loss_valid += loss_func(pred_valid, yb_valid)\n            print(\"Accuracy of validation set :\", valid_acc/len(valid_dl))\n\n    plt.plot(losses)\n    plt.ylabel(\"loss\")\n    plt.xlabel(\"iterations (per hundreds)\")\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = get_data(train_ds, bs=64)\nvalid_dl = get_data(valid_ds, bs = 128)\ntrain_dl = WrappedDataLoader(train_dl, preprocess)\nvalid_dl = WrappedDataLoader(valid_dl, preprocess)\n\nopt = optim.Adam(model.parameters(), lr=0.1)\nlr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(opt, patience=1, verbose=True, eps = 10e-10)\n\nfit(model, 160, train_dl, valid_dl, opt)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Deleting some variables of no use later to free some space\ndel train_dl, valid_dl, x_train, y_train, x_cv, y_cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = x_test/255.0\nx_test = x_test.reshape(-1, 1, 28, 28)\nwith torch.no_grad():\n    x_test = x_test    #Passing the entire test set to the GPU\n    test_out = model(x_test.float()).cpu()\n    test_pred = torch.argmax(test_out, dim = 1, keepdim=True).cpu().numpy()\n    #test_pred_np = test_pred.cpu().numpy() \n    test_pred = np.reshape(test_pred, test_pred.shape[0])\n    print(test_pred.shape)\n    \n    row1 = pd.Series(test_pred, name=\"Label\")\n    row2 = pd.Series(range(1, 28001), name=\"ImageId\")\n    submission = pd.concat([row2, row1], axis=1)\n    submission.to_csv(\"cnn_mnist_datagen.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_accuracy(model):\n    print(\"Accuracy of train set:\", accuracy(model(x_train.float()), y_train))\n    print(\"Accuracy of test set:\", accuracy(model(x_cv.float()), y_cv))\n\nprint_accuracy(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}