{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nfrom PIL import Image\nimport time\nimport errno\nimport torch\nimport torch.nn.functional as F\nimport matplotlib.pyplot as plt\nfrom torchvision import models,datasets,transforms\nfrom torch import nn,optim\nfrom torch.utils.data.sampler import SubsetRandomSampler\nfrom collections import OrderedDict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Genral Preprocesing of csv files**\n\nThe pixel values can be taken and fed into a simple Feed Forward ANN, but in this case we are going to use a CNN. Hence one of the ways to use PyTorch's Dataloader function is to write the images into files and store each individual file into their corresponding folder.\n\nFor example in this case the folder structures will be like this\n\n* myData\n*       train\n            ->0\n            ->1\n            ->2\n            ->3\n            .\n            .\n            .\n            ->9\n*       test\n            ->0\n            ->1\n            ->2\n            ->3\n            .\n            .\n            .\n            ->9\n            \nTo load each images into its corresponding folder\n1. Read the csv files into pandas dataframe\n2. Convert the dataframe into numpy array\n3. Select each row:\n    a. First element will be 'class id' and rest will be pixel values for 28 x 28 image\n    b. Generate image from array[1:] into a 28x28 size image and pase it into folder 'train/class_id'\n\n\nFor generating the Train, Validation, and Test sets:\n1. Select 80% of total training data as trian set and remaining 20% as validation set\n2. Select the whole test folder data for test set. (Test data is saved in 'Dig-MNIST.csv'\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Select GPU if available\ndevice='cpu'\nif(torch.cuda.is_available()):\n    device='cuda'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extract_folder_location='/kaggle/input/Kannada-MNIST/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=pd.read_csv(extract_folder_location+'train.csv') #Train data csv file\ndf_test=pd.read_csv(extract_folder_location+'Dig-MNIST.csv') # Test data csv file","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=df_train.to_numpy() #convert both dataframes to numpy array to iterate over each row seperately\ndf_test=df_test.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.mkdir('myData')      \nos.mkdir('myData/train')  # Create the necessary train folder\nfor i in range(10):\n    os.mkdir('myData/train/'+str(i)) # since Image is divided into 10 classes, generate 10 distinct folder for each class images\n    \nos.mkdir('myData/test')\nfor i in range(10):\n    os.mkdir('myData/test/'+str(i))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generateImage(imageArray,imageWidth,imageHeight):\n    '''\n    Parameters :\n        imageArray: numpy 1d array with pixel intensity values\n        imageWidth: Width of target image\n        imageHeight: Height of target image\n    \n    Returns: Generated grey scale PIL image with resolution Width x Height \n    '''\n    image=np.zeros(shape=(imageWidth,imageHeight))\n    index=0\n    for i in range(imageWidth):\n        for j in range(imageHeight):\n            image[i][j]=imageArray[index]\n            index+=1\n    img=Image.fromarray(image)\n    img=img.convert(\"L\")\n    return img","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to save PIL image to file\ndef saveImage(image,saveLocation):\n    image.save(saveLocation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to iterate over whole train dataset, convert each row of pixel values into iamge and save the image into its corresponding class id folder\nfor i in range(len(df_train)):\n    saveImage(generateImage(df_train[i][1:],28,28),'myData/train/'+str(df_train[i][0])+'/'+str(i)+'.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to iterate over whole test dataset, convert each row of pixel values into iamge and save the image into its corresponding class id folder\nfor i in range(len(df_test)):\n    saveImage(generateImage(df_test[i][1:],28,28),'myData/test/'+str(df_test[i][0])+'/'+str(i)+'.png')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dir='myData'  # Train and test data location directory\nnum_workers=0      # This is set to 0 becasue we are only using 1 hardware accelerator\nbatch_size=128     \nvalid_size=0.2     # Fraction of data that will be taken from training data for validation set\n\n\n# Image transformations for greay scale image, to convert it into a tensor and normalize it\ntrain_transforms = transforms.Compose([transforms.Grayscale(num_output_channels=1),\n                                       transforms.ToTensor(),\n                                       transforms.Normalize([0.5],\n                                                            [0.5])])\n\ntest_transforms = transforms.Compose([transforms.Grayscale(num_output_channels=1),\n                                      transforms.ToTensor(),\n                                      transforms.Normalize([0.5],\n                                                            [0.5])])\n\ntrain_data = datasets.ImageFolder(data_dir + '/train', transform=train_transforms) #Use PyTorch datasets.ImageFolder to load and transform images into tensor\ntest_data = datasets.ImageFolder(data_dir + '/test', transform=test_transforms)\n\n\n# Split the train set into train and validation set and shuffle it\nnum_train = len(train_data)   \nindices = list(range(num_train))\nnp.random.shuffle(indices)\nsplit = int(np.floor(valid_size * num_train))\ntrain_idx, valid_idx = indices[split:], indices[:split]\n\ntrain_sampler = SubsetRandomSampler(train_idx)\nvalid_sampler = SubsetRandomSampler(valid_idx)\n\n\n# Put the data into dataloader\ntrain_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size,\n    sampler=train_sampler, num_workers=num_workers)\nvalid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, \n    sampler=valid_sampler, num_workers=num_workers)\n\ntest_loader = torch.utils.data.DataLoader(test_data,batch_size=batch_size,num_workers=num_workers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_loader),len(valid_loader),len(test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the class to id mapping \nprint(train_data.class_to_idx)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Custom CNN Model**\n\nFor this custom CNN we will define a class for CNN\n\nInput Image size is 28 x 28 with 1 channel\n\n**First Convolution Layer:** Takes 28x28 image, with 3x3 kernel and 1 stride and padding\n**Second Average Pooling Layer:** Takes 28x28 image with 50 channels, runs kernel of size 2x2 with stride of 2, basically trimming down the image dimension to half,\n                              Hence the output will be : 14x14 image with 50 channel\n                              \nSubsiquent Linear Layers with last layer passed with Log Softmax activation function.\nFirst linear layer will have input of size [512 x (50x14x14)]\n\n    \n    \n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"class KannadaClassifierCNN(torch.nn.Module):\n    def __init__(self):\n        super(KannadaClassifierCNN,self).__init__()\n        \n        self.conv1 = torch.nn.Conv2d(1,50,kernel_size=3,stride=1,padding=1)\n        self.pool = torch.nn.MaxPool2d(kernel_size=2,stride=2,padding=0)\n        self.fc1 = torch.nn.Linear(50*14*14,512)\n        self.fc2 = torch.nn.Linear(512,256)\n        self.fc3 = torch.nn.Linear(256,128)\n        self.fc4 = torch.nn.Linear(128,64)\n        self.fc5 = torch.nn.Linear(64,10)\n        self.dropout = torch.nn.Dropout(0.25)\n        \n        # Function to make forward pass into CNN\n    def forward(self,x):\n        x = F.relu(self.conv1(x))\n        x = F.relu(self.pool(x))\n        x = F.relu(self.fc1(x.view(-1,50*14*14)))\n        x = self.dropout(x)\n        x = F.relu(self.fc2(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc3(x))\n        x = self.dropout(x)\n        x = F.relu(self.fc4(x))\n        x = self.fc5(x)\n        return F.log_softmax(x,dim=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def createModel():\n    model=KannadaClassifierCNN()\n    \n    # Unfreeze all parameters to allow gradient calculation for backpropagate\n    for param in model.parameters():\n        param.requires_grad=True\n        \n    # Loss and optimizer for model\n    criterion = nn.CrossEntropyLoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.0001)\n    return model,criterion,optimizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def trainNetwork(model,epochs):\n    # Passes model has be already on GPU if GPU was available\n    train_on_gpu=False\n    if device=='cuda':\n        train_on_gpu=True\n    n_epochs = epochs\n    valid_loss_min = np.Inf  # Set initial validation loss to MAX \n    Training_Loss=[]\n    Validation_Loss=[]\n    Iteration=[]\n    print('Train on gpu is : ',train_on_gpu)\n\n\n    for epoch in range(1, n_epochs+1):\n\n        train_loss = 0.0\n        valid_loss = 0.0\n        Iteration.append(epoch+1)\n\n        model.train()\n        for data, target in train_loader:\n\n            if train_on_gpu:   # GPU is available then move data to GPU and train\n                data, target = data.cuda(), target.cuda()\n\n            optimizer.zero_grad()\n            output = model(data)\n            loss = criterion(output, target)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()*data.size(0)\n\n\n        model.eval()\n        for data, target in valid_loader:\n\n            if train_on_gpu:\n                data, target = data.cuda(), target.cuda()\n\n            output = model(data)\n            loss = criterion(output, target)\n            valid_loss += loss.item()*data.size(0)\n\n        train_loss = train_loss/len(train_loader.dataset)\n        valid_loss = valid_loss/len(valid_loader.dataset)\n        Training_Loss.append(train_loss)\n        Validation_Loss.append(valid_loss)\n\n\n        print('Epoch: {} \\tTraining Loss: {:.6f} \\tValidation Loss: {:.6f}'.format(\n          epoch, train_loss, valid_loss))\n\n        if valid_loss <= valid_loss_min: #Each time validation loss decreases save the model\n            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(\n          valid_loss_min,\n          valid_loss))\n            torch.save(model.state_dict(), 'checkpointFinal.pt')\n            saveModel(model,'CheckpointFinal2.pth')\n            valid_loss_min = valid_loss\n\n    plt.plot(Iteration,Training_Loss)\n    plt.plot(Iteration,Validation_Loss)\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.ylim(0.0,1.0)\n    plt.xlim(0,epochs)\n    plt.legend(['Training Loss','Validation Loss'], loc='upper left')\n    plt.show()\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def saveModel(model,path):\n    model.class_to_idx = train_data.class_to_idx\n    checkpoint = {'input_size': 28*28,\n                'output_size': 10,\n                'model': model,\n                'state_dict': model.state_dict(),\n                'optimizer_state_dict': optimizer.state_dict,\n                'criterion': criterion,\n                'class_to_idx':model.class_to_idx\n               }\n    torch.save(checkpoint, path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to save model checkpoint\ndef load_checkpoint(filepath):\n    checkpoint=torch.load(filepath)\n    model=checkpoint[\"model\"]\n    model.class_to_idx = checkpoint['class_to_idx']\n    model.load_state_dict(checkpoint['state_dict'],strict=False)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to check accuracy by passing model and corresponding dataloader (it can be train_loader, validation_loader or test_loader)\ndef check_accuracy(model,testloader):    \n    correct = 0\n    total = 0\n    model.to('cuda')\n    with torch.no_grad():\n        for data in testloader:\n            images, labels = data\n            images, labels = images.to('cuda'), labels.to('cuda')\n            outputs = model(images)\n            _, predicted = torch.max(outputs.data, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    print('Accuracy of the network on the test images: %d %%' % (100 * correct / total))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# For single image prediction , image processing function\ndef process_image(img_pil):\n\n    adjustments = transforms.Compose([transforms.ToTensor(),\n        transforms.Normalize([0.5],[0.5])\n    ])\n    \n    img_tensor = adjustments(img_pil)\n    \n    return img_tensor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction function to reutrn the top k prob for classes \ndef predict(image_path, model, topk=1):   \n    model.to(device)\n    img_torch = process_image(image_path)\n    img_torch = img_torch.unsqueeze_(0)\n    img_torch = img_torch.float()\n    \n    with torch.no_grad():\n        output = model.forward(img_torch.cuda())\n        \n    probability = F.softmax(output.data,dim=1)\n    \n    return probability.topk(topk)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Final Steps**\n1. Create model\n2. Move model to GPU\n3. Initiate training for some epochs\n4. Study the loss graph and check accuracy\n5. Train for more epochs with smaller leanring rate by reinitializing the optimizer\n6. Load the final saved checkpoing and initiate prediction function for each submission file\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"model,criterion,optimizer=createModel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=trainNetwork(model,10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_accuracy(model,test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=trainNetwork(model,5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_accuracy(model,test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = optim.Adam(model.parameters(), lr=0.000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=trainNetwork(model,45)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=load_checkpoint('CheckpointFinal2.pth')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_accuracy(model,test_loader)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission=pd.read_csv(extract_folder_location+'sample_submission.csv')\ndf_submission_mapping=pd.read_csv(extract_folder_location+'test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission_mapping.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission_mapping=df_submission_mapping.to_numpy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp=[]\nfor i in range(len(df_submission_mapping)):\n    image=generateImage(df_submission_mapping[i][1:],imageWidth=28,imageHeight=28)\n    img_torch = process_image(image)\n    img_torch = img_torch.unsqueeze_(0)\n    img_torch = img_torch.float()\n    outputs=model.forward(img_torch.cuda())\n    _, predicted = torch.max(outputs.data, 1)\n    temp.append(predicted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result=[]\nfor element in temp:\n    result.append(element.item())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission['label']=result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_submission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Remove all generated files otherwise Kaggle will show error with commit that \"More than 500 outputs were generated\""},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm -rf myData","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm CheckpointFinal2.pth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!rm checkpointFinal.pt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}