{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\n# import torchvision module to handle image manipulation\nimport torchvision\nimport torchvision.transforms as transforms\nfrom torch.utils.data import Dataset, DataLoader\n# calculate train time, writing train data to files etc.\nimport time\nimport pandas as pd\nimport json\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"class Dataloader(Dataset):\n    \n    def __init__(self,image,label,is_train=True):\n        \n        self.img = image\n        self.label = label\n        self.is_train = is_train\n        \n    def __len__(self):\n        return len(self.img)\n    \n    def __getitem__(self,idx):\n        \n        '''\n        Reshape: 1D) 784 -> 2D) 28x28\n        '''\n        image1 = self.img[idx].reshape(-1,28,28)\n        if self.is_train:\n            label1 = np.zeros(10, dtype='float32')\n            label1[self.label[idx]] =1\n            #label1 = self.label\n            return image1,label1\n        else:\n            return image1\n        \n\ntrain_data = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ntrain_data = train_data.sample(frac=1,random_state = 42)\ntest_data = train_data[55000:]\n\n\ntrain_data = train_data[0:54999]\n\ntrain_label = train_data['label'].values\nprint(train_label)\ntrain_matrix = train_data.drop('label',axis=1).values/255\n\ntrainset = Dataloader(train_matrix,train_label)\n\n## Create Test Set\n\ntest_label = test_data['label'].values\ntest_matrix = test_data.drop('label',axis=1).values/255\ntestset = Dataloader(test_matrix,test_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## Display the images\ndisplay_loader = torch.utils.data.DataLoader(trainset,batch_size=50)\nbatch = next(iter(display_loader))\nimages,label = batch\nimport matplotlib.pyplot as plt\ngrid = torchvision.utils.make_grid(images,nrow=10)\nplt.figure(figsize=(15,15))\nplt.imshow(np.transpose(grid,(1,2,0)))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## Declare a network object\n\nclass Network(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels = 1,out_channels = 3,kernel_size=5)\n        self.conv2 = nn.Conv2d(in_channels= 3 , out_channels = 6,kernel_size = 5)\n        self.fc1 = nn.Linear(in_features=4*4*6,out_features=120)\n        self.fc2 = nn.Linear(in_features=120,out_features = 60)\n        self.out = nn.Linear(in_features = 60,out_features=10)\n        \n    def forward(self,t):\n        ## Forward Pass\n        ## Input Layer\n        t = t\n        ## First Convolutional Layer\n        t = self.conv1(t)\n        t = F.relu(t)\n        t = F.max_pool2d(t,kernel_size=2,stride=2)\n        print('Shape After 1st Convolution and max pooling')\n        print(t.shape)\n        ## Second Convolutional Layer\n        t = self.conv2(t)\n        t = F.relu(t)\n        t = F.max_pool2d(t,kernel_size=2,stride=2)\n\n        print('Shape After second convolutional network')\n        print(t.shape)\n        \n        ## Flattened hidden layer\n        t = t.reshape(-1,4*4*6)\n        t = self.fc1(t)\n        t = F.relu(t)\n        print('Shape After 1st FC Network')\n        print(t.shape)\n        \n        ## Second Hidden layer\n        \n        t = self.fc2(t)\n        t = F.relu(t)\n        print('Shape After 2nd FC Network')\n        print(t.shape)\n        \n        ## Output Layer\n        \n        t = self.out(t)\n        print('Shape After Last FC Network')\n        print(t.shape)\n        \n        ## Softmax is not used because F.cross_entropy() function implicitly performs the softmax operation.\n        \n        \n        \n        \n        return(t)\n        \ndef get_num_correct(pred,labels):\n    return(pred.argmax(dim=1).eq(labels.argmax(dim=1)).sum().item())        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## Training and Testing with hyper parameter tuning\n## Create a list of Hyper Parameters\n\nfrom itertools import product\n\nparameters = dict(\n    lr = [0.01,0.001],\n    batch_size = [100,1000],\n    shuffle = [True,False]\n    \n)\nparam_values = [v for v in parameters.values()]\nparam_values\n\nfor lr,batch_size,shuffle in product(*param_values):\n    print(lr,batch_size,shuffle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"## Unpack the parameters\n## Run the \nresults = {}\nimport numpy as np\nc = 0\n\nfor lr,batch_size,shuffle in product(*param_values):\n    network = Network()\n    train_loader = torch.utils.data.DataLoader(trainset,batch_size = batch_size,shuffle = shuffle)\n    test_loader = torch.utils.data.DataLoader(testset,batch_size=batch_size,shuffle = shuffle)\n    optimizer = optim.Adam(network.parameters(),lr=lr)\n    images, labels = next(iter(train_loader))\n    for epoch in range(5):\n\n        print(epoch)\n        total_train_loss = 0\n        total_train_correct = 0\n        total_test_loss = 0\n        total_test_correct = 0\n        \n        for batch in train_loader:\n            images,labels = batch\n#             print(images.shape)\n#             print(labels.shape)\n            pred = network(images.float())\n#             print(pred.shape)\n            loss = F.cross_entropy(pred,labels.argmax(dim=1))\n            optimizer.zero_grad() # flush gradients\n            loss.backward() # Calculate Gradients\n            optimizer.step() # update weight\n            total_train_loss = total_train_loss + (loss.item()*images.shape[0]) ## Total Loss\n            total_train_correct = total_train_correct + get_num_correct(pred,labels)\n            \n        if epoch%1==0:\n            with torch.no_grad():\n                for batch in test_loader:\n                    images,labels = batch\n                    pred = network(images.float())\n                    loss = F.cross_entropy(pred,labels.argmax(dim=1))\n                    total_test_loss = total_test_loss + (loss.item()*images.shape[0])\n                    total_test_correct = total_test_correct + get_num_correct(pred,labels)\n            result_dict = {\n                'epoch' : epoch,\n                'total_train_loss' : total_train_loss,\n                'total_train_correct': total_train_correct,\n                'total_test_loss' : total_test_loss,\n                'total_test_correct': total_test_correct,\n                'batch_size': batch_size,\n                'lr': lr\n            }\n            results[c] = result_dict\n            c = c+1\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"correct_train = []\nmax_index = 0\nmax_test_accuracy = 0\nfor i in range(8):\n    ## Get the index with the highest test accuracy and \n    if results[i]['total_test_correct'] > max_test_accuracy:\n        max_index = i\n        max_test_accuracy = results[i]['total_test_correct']\n    \n    correct_train.append(results[i]['total_test_correct'])\n\nimport matplotlib.pyplot as plt\nplt.plot(correct_train)\nplt.xlabel('')\nplt.ylabel('Test Accuracy')\nplt.show()\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"correct_train = []\nfor i in range(8):\n    correct_train.append(results[i]['total_train_correct'])\n\nimport matplotlib.pyplot as plt\nplt.plot(correct_train)\nplt.xlabel('')\nplt.ylabel('Train Accuracy')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\nresults[max_index]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\ntest_img = test.iloc[:,1:].astype('float').values/255.0\ntestset = Dataloader(test_img,None,is_train=False)\ntest_loader = torch.utils.data.DataLoader(testset,batch_size=100,shuffle=False)\n## Re create Model with highest Test score\nnetwork = Network()\ntrain_loader = torch.utils.data.DataLoader(trainset,batch_size = results[max_index]['batch_size'],shuffle = shuffle)\noptimizer = optim.Adam(network.parameters(),lr=results[max_index]['lr'])\nfor epoch in range(5):\n    for batch in train_loader:\n        images,labels = batch\n        pred = network(images.float())\n        loss = F.cross_entropy(pred,labels.argmax(dim=1))\n        optimizer.zero_grad() # flush gradients\n        loss.backward() # Calculate Gradients\n        optimizer.step() # update weight\n\n\n\n\n\npredictions = []\n\nfor batch in test_loader:\n    images = batch\n    preds = network(images.float())\n    predictions += list(preds.argmax(dim=1).cpu().detach().numpy())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/Kannada-MNIST/sample_submission.csv')\nsubmission['label'] = predictions\nsubmission.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"for grp in optimizer.param_groups:\n    print(grp['lr'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}