{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<h1 align=center><font size=5>Digit Recognition. CNN in PyTorch. </font></h1>","metadata":{}},{"cell_type":"code","source":"#Needed libraries and modules\nimport pandas as pd                                  # data processing, reading CSV file, dealing with dataframes etc\nimport numpy as np                                   # linear algebra functionalities \nimport matplotlib.pyplot as plt                      # visualization library\n\n#Modeling\nfrom sklearn.model_selection import train_test_split, GridSearchCV, cross_val_score\n\nimport math \nimport time\nimport multiprocessing\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import datasets\nfrom torchvision.transforms import ToTensor, Lambda, Compose\nfrom torchvision import transforms\ntorch.manual_seed(2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Table of Contents\n\n* [1. Loading Data](#Section1)\n    * [1.1 From cvs files to dataframes](#SubSection_1_1)\n    * [1.2 Dataset class and transforms](#SubSection_1_2)\n    * [1.3 Train and validation sets. Visualization](#SubSection_1_3)\n* [2. CNN with Batch Normalization](#Section2)\n* [3. Training the model](#Section3)\n* [4 Analyzing results](#Section4)\n* [5 Predictions on test set](#Section5)","metadata":{}},{"cell_type":"markdown","source":" # 1. Loading Data <a class=\"anchor\" id=\"Section1\"></a>","metadata":{}},{"cell_type":"markdown","source":"We want to be able to train our model on a hardware accelerator like the GPU, \nif it is available. Let's check to see if \n\"torch.cuda\" <https://pytorch.org/docs/stable/notes/cuda.html> is available, else we \ncontinue to use the CPU. ","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint('Using {} device'.format(device))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.1 Loading data from cvs files into data frames<a class=\"anchor\" id=\"SubSection_1_1\"></a> ","metadata":{}},{"cell_type":"code","source":"#Loading training and test set to dataframes\npath_train = '../input/digit-recognizer/train.csv'\npath_test = '../input/digit-recognizer/test.csv'\n\ndf_train = pd.read_csv(path_train)\ndf_test = pd.read_csv(path_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We convert the data to numpy. We normalize the data to avoid the vanishing gradient problem. This might be the case if one trains the model without usign batch normalzation.","metadata":{}},{"cell_type":"code","source":"#Training data\nX = df_train.iloc[:, 1:].to_numpy() /255\nY = df_train.iloc[:, 0].to_numpy()\n\n#Train set & validation sets\nX_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.1, random_state=1)\n\n#Test set \nX_test = df_test.to_numpy() / 255\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Shape of train dataset: ', X_train.shape)\nprint('Shape of validation dataset: ', X_val.shape)\nprint('Shape of test dataset: ', X_test.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 1.2 Dataset class and transforms<a class=\"anchor\" id=\"SubSection_1_2\"></a> ","metadata":{}},{"cell_type":"markdown","source":"Datset class","metadata":{}},{"cell_type":"code","source":"class Dataset(Dataset):\n    \n    def __init__(self, x, y = None, transform = None):\n        self.x = x\n        self.y = y\n        self.transform = transform\n        self.lenght = self.x.shape[0]\n        \n    def __len__(self):\n        return self.lenght\n    \n    def __getitem__(self, idx):\n        image = self.x[idx, :]\n        if self.transform:\n            image = self.transform(image)\n        label = int(self.y[idx])\n        return image, label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transforms","metadata":{}},{"cell_type":"code","source":"class Numpy_to_ImageTensor_Reshape(object):\n    '''\n    This is a transform object, that transforms a given numpy array into a tensor with dimensions especified by dims.\n    '''\n    # Constructor\n    def __init__(self, dims):\n        self.dims = dims\n        \n    # Executor\n    def __call__(self, sample):\n        image = torch.from_numpy(sample).type(torch.FloatTensor)\n        image = image.view(1,self.dims[0], self.dims[1])\n        return image","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"IMAGE_SIZE = 28\ncomposed = transforms.Compose([Numpy_to_ImageTensor_Reshape([28,28]), transforms.Resize((IMAGE_SIZE,IMAGE_SIZE))])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train and test set","metadata":{}},{"cell_type":"code","source":"train_set = Dataset(x=X_train, y=Y_train, transform=composed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation_set = Dataset(x=X_val, y=Y_val, transform=composed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of train set: ', len(train_set))\nprint('Size of validation set: ', len(validation_set))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Function to print images:","metadata":{}},{"cell_type":"code","source":"def show_data(data_sample):\n    image = data_sample[0].to('cpu')\n    plt.imshow(image.view(IMAGE_SIZE,IMAGE_SIZE), cmap='gray')\n    plt.title('y = '+ str(data_sample[1]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_data(train_set[9])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_data(validation_set[156])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # 2. CNN with Batch Normalization <a class=\"anchor\" id=\"Section2\"></a>","metadata":{}},{"cell_type":"code","source":"class CNN_BatchNorm(nn.Module):\n     \n    # Contructor\n    def __init__(self, Conv_Layers, FC_Layers, BatchNorm = False):\n        super(CNN_BatchNorm, self).__init__()\n        \n        self.Conv = nn.ModuleList()\n        self.FC = nn.ModuleList()\n        \n        #Building convolution layers\n        for layer in Conv_Layers:\n            conv = nn.Conv2d(in_channels=layer[0], out_channels=layer[1], kernel_size=layer[2], \n                             stride= layer[3], padding=layer[4])\n            maxpool = nn.MaxPool2d(kernel_size=layer[5])\n            self.Conv.append(conv)\n            if BatchNorm: \n                self.Conv.append(nn.BatchNorm2d(layer[1]))\n            self.Conv.append(nn.ReLU())\n            self.Conv.append(maxpool)\n            \n        #Building Fully connected layers\n        L = len(FC_Layers)\n        for i in range(L-1):\n            input_size = FC_Layers[i]\n            output_size = FC_Layers[i+1]\n            self.FC.append(nn.Linear(input_size, output_size))\n            if BatchNorm:\n                self.FC.append(nn.BatchNorm1d(output_size))\n            if i < L-2:\n                self.FC.append(nn.ReLU())\n            \n    #Prediction\n    def forward(self, x):\n        #Convolutional layers:\n        for module in self.Conv:\n            x = module(x)\n            \n        #Flattening\n        x = x.view(x.size(0), -1)\n       \n        #Fully connected layers\n        for transform in self.FC:\n            x = transform(x)\n        return x","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # 3. Training the model <a class=\"anchor\" id=\"Section3\"></a>","metadata":{}},{"cell_type":"markdown","source":"Function to train the model","metadata":{}},{"cell_type":"code","source":"def train_model(model, train_loader, validation_loader, optimizer, criterion=nn.CrossEntropyLoss(), n_epochs=4):\n    #global variable \n    N_test=len(validation_set)\n    \n    accuracy_list=[]\n    loss_list=[]\n    \n    for epoch in range(n_epochs):\n        #Training\n        COST = 0\n        model.train()   #batch normalization in training mode\n        for x, y in train_loader:\n            x, y = x.to(device), y.to(device)\n            optimizer.zero_grad()\n            z = model(x)\n            loss = criterion(z, y)\n            loss.backward()\n            optimizer.step()\n            COST += loss.data\n        loss_list.append(COST)\n        \n        #Validating\n        correct=0\n        model.eval()    #batch normalization in validation mode\n        for x_test, y_test in validation_loader:\n            x_test, y_test = x_test.to(device), y_test.to(device)\n            z = model(x_test)\n            _, yhat = torch.max(z.data, 1)\n            correct += (yhat == y_test).sum().item()\n        accuracy = correct / N_test\n        accuracy_list.append(accuracy)\n     \n    return accuracy_list, loss_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will now do a grid search looking for good parameters of the convolution kernel_size in the model and for the momentum parameter in the GD algorithm. \n\nWe will always use 2 convolutional layers and one fully connected layer. The general configuration is as follows:\n\n* 1st convolutional layer: [in_channels = 1, out_channels = 16, convolution_kernel_size = 2*n+1, stride= 1, padding = n, maxpool__kernel_size =2]\n* 1st convolutional layer: [in_channels = 16, out_channels = 32, convolution_kernel_size = 2*n+1, stride= 1, padding = n, maxpool__kernel_size =2]\n* Fully connected layer: [32 * 7 * 7,10]\n\nLet's now define the fnction for grid search\n","metadata":{}},{"cell_type":"code","source":"def GridSearch(ker_size_list = [5], momentum_list = [0]):\n    Results = {}\n    Best_accuracy = 0\n    Best_model = None\n    Best_parameters = {'kernel_size': None, 'momentum': None}\n    \n    for k in ker_size_list:\n        for momentum in momentum_list:\n            model = CNN_BatchNorm([[1,16,2*k+1,1,k,2], [16,32,2*k+1,1,k,2]], [32*7*7,10], BatchNorm = True)\n            model.to(device)\n            learning_rate = 0.10\n            optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=momentum)\n            Results[\"K_\"+str(k)+\" & M_\"+str(momentum)] = train_model(model=model, train_loader=train_loader,\n                                                                   validation_loader=validation_loader, \n                                                                   optimizer=optimizer, n_epochs=n_epochs)\n            if Best_accuracy < Results[\"K_\"+str(k)+\" & M_\"+str(momentum)][0][n_epochs-1]: \n                Best_accuracy = Results[\"K_\"+str(k)+\" & M_\"+str(momentum)][0][n_epochs-1]\n                Best_model = model\n                Best_parameters['kernel_size'] = k\n                Best_parameters['momentum'] = momentum\n    \n    return Best_accuracy, Best_parameters, Best_model, Results","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Loss function and data loaders","metadata":{}},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\nn_epochs=30\ntrain_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=100)\nvalidation_loader = torch.utils.data.DataLoader(dataset=validation_set, batch_size=5000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GridSearch","metadata":{}},{"cell_type":"code","source":"%%time\nker_size_list = [5, 6]\nmomentum_list = [0.0, 0.5, 0.8]\n\nBest_accuracy, Best_parameters, Best_model, Results = GridSearch(ker_size_list = ker_size_list,\n                                                               momentum_list = momentum_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Best accuracy: ', Best_accuracy)\nprint('Best parameters: ', Best_parameters)\nprint('Best model: \\n', Best_parameters)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # 4. Analizing results <a class=\"anchor\" id=\"Section4\"></a>","metadata":{}},{"cell_type":"code","source":"# Plot the Loss result for each term\n\nfor key, value in Results.items():\n    plt.plot(value[1],label=key)\n    plt.legend()\n    plt.xlabel('epoch')\n    plt.ylabel('Total Loss or Cost')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the Loss result for each term\n\nfor key, value in Results.items():\n    plt.plot(value[0],label=key)\n    plt.legend()\n    plt.xlabel('epoch')\n    plt.ylabel('Accuracy or Cost')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the mis-classified samples\n\ncount = 0\nfor x, y in torch.utils.data.DataLoader(dataset=validation_set, batch_size=1):\n    x, y = x.to(device), y.to(device)\n    z = Best_model(x)\n    _, yhat = torch.max(z, 1)\n    if yhat != y:\n        show_data((x, y))\n        plt.show()\n        print(\"yhat: \",yhat)\n        count += 1\n    if count >= 5:\n        break  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" # 5. Predictions on the test set <a class=\"anchor\" id=\"Section5\"></a>","metadata":{}},{"cell_type":"code","source":"#Test set \nX_test = df_test.to_numpy() / 255\n\nX_test = torch.from_numpy(X_test).type(torch.FloatTensor).view(-1,1,28,28).to(device)\nz = Best_model(X_test)\n_, yhat_test = torch.max(z, 1)\nyhat_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\nsample_sub['Label'] = yhat_test.to('cpu')\nsample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.to_csv('CNN_submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}