{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#cnn params\n\ninit_width = 32\n#dataloader params\nbatch_size = 100\nl_rate = 0.001\nmomentum = 0.0\nweight_decay = 0.0\ndampening = 0.0\nnesterov = 0 #0 or 1\nlr_gamma = 0.9\nepochs = 100\n#transforms params\ntrans_probability = 0.5\nmax_rotation_right = 9\nmax_rotation_left = 9\nmax_factor = 1.15\nmin_factor = 1.05\ngrid_height = 5\ngrid_width = 5\nmagnitude = 3\nbrightness = 2\ncontrast   = 2\ntranslate  = 0.1\nblur_max = 3\nblur_min = 0.5\nblur_kernel = 5\nopt_kwargs = dict(batch_size=batch_size,\n                  l_rate=l_rate,\n                  lr_gamma=lr_gamma,\n                  momentum=momentum,\n                  weight_decay=weight_decay,\n                  dampening=dampening,\n                  nesterov=nesterov,\n                  probability=trans_probability,\n                  max_rotation_right=max_rotation_right,\n                  max_rotation_left=max_rotation_left,\n                  max_factor=max_factor,\n                  min_factor=min_factor,\n                  grid_height=grid_height,\n                  grid_width=grid_width,\n                  magnitude=magnitude,\n                  init_width=init_width,\n                  brightness=brightness,\n                  contrast = contrast,\n                  translate=translate,\n                  blur_kernel = blur_kernel,\n                  blur_min = blur_min,\n                  blur_max = blur_max,\n                \n                  )\noptimizer = 'adam'","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:07.508384Z","iopub.execute_input":"2022-10-23T21:20:07.508810Z","iopub.status.idle":"2022-10-23T21:20:07.547193Z","shell.execute_reply.started":"2022-10-23T21:20:07.508725Z","shell.execute_reply":"2022-10-23T21:20:07.546264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Let's see what we have in our imported directory\n! ls ../input/augmentor/Augmentor","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:07.549135Z","iopub.execute_input":"2022-10-23T21:20:07.549550Z","iopub.status.idle":"2022-10-23T21:20:08.495812Z","shell.execute_reply.started":"2022-10-23T21:20:07.549510Z","shell.execute_reply":"2022-10-23T21:20:08.494718Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install Augmentor --no-index --find-links=file:///kaggle/input/augmentor/Augmentor","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:08.497811Z","iopub.execute_input":"2022-10-23T21:20:08.498228Z","iopub.status.idle":"2022-10-23T21:20:19.773289Z","shell.execute_reply.started":"2022-10-23T21:20:08.498187Z","shell.execute_reply":"2022-10-23T21:20:19.772162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\ntorch.__version__","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:19.777668Z","iopub.execute_input":"2022-10-23T21:20:19.777973Z","iopub.status.idle":"2022-10-23T21:20:21.931438Z","shell.execute_reply.started":"2022-10-23T21:20:19.777942Z","shell.execute_reply":"2022-10-23T21:20:21.930319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn\nimport torchvision.transforms.functional as TF\nimport random\nimport torchvision.transforms as transforms\nimport Augmentor,os\nimport numpy as np\nfrom Augmentor.Operations import Distort\n\n\nclass RandomDistortion:\n    def __init__(self,probability, grid_width, grid_height, magnitude):\n        self.probability = probability\n        self.grid_width  = grid_width\n        self.grid_height = grid_height\n        self.magnitude   = magnitude\n        self.op          = Distort(probability, grid_width, grid_height, magnitude)\n\n    def __call__(self,x):\n        #x =\n        #print(x[0].size)\n        out = self.op.perform_operation([x])\n\n\n\n        return out[0]\n\n    def __repr__(self):\n        return self.__class__.__name__+'(probability={:},grid_width={:},grid_height={:},magnitude={:})'.format(\n            self.probability,\n                                                                                        self.grid_width,\n                                                                       self.grid_height,\n                                                                   self.magnitude)\n\nclass SaltPepperNoise:\n    \"\"\"Rotate by one of the given angles.\"\"\"\n\n    def __init__(self, mean=0,std=1):\n        self.mean = mean\n        self.std  = std\n\n    def __call__(self, x):\n\n        return x + np.random.randn(*x.size) * self.std + self.mean\n\n    def __repr__(self):\n        return self.__class__.__name__+'(mean={:},std={:})'.format(self.mean,self.std)\n\nclass MapToInterval:\n    def __init__(self):\n        self.min=0\n        self.max=1\n\n\n    def __call__(self,x):#image = (im - np.min(im)) / (np.max(im) - np.min(im))\n        #https://stackoverflow.com/questions/72440228/how-to-rescale-a-pytorch-tensor-to-interval-0-1\n        flattened_outmap = x.view(x.shape[0],-1,1,\n                                       1)  # Use 1's to preserve the number of dimensions for broadcasting later,\n        # as explained\n        outmap_min,_ = torch.min(flattened_outmap,dim=1,keepdim=True)\n        outmap_max,_ = torch.max(flattened_outmap,dim=1,keepdim=True)\n        outmap = (x - outmap_min) / (outmap_max - outmap_min)  # Broadcasting rules apply\n\n        return outmap\n\n    def __repr__(self):\n        return self.__class__.__name__+'(min={:},max={:})'.format(self.min,self.max)\n\n\n\n#rotation_transform = MyRotationTransform(angles=[-30, -15, 0, 15, 30])\n\n\n\ndef create_train_test_transform(log_dir,write_params=True,kwargs={}):\n\n    probability       = kwargs.get(\"probability\",1)\n    max_rotation_left = kwargs.get(\"max_rotation_left\",10.0)\n    max_rotation_right = kwargs.get(\"max_rotation_right\",10.0)\n\n    max_factor = kwargs.get(\"max_factor\",1.1)\n    min_factor = kwargs.get(\"min_factor\",1.05)\n\n    grid_height = kwargs.get(\"grid_height\",15)\n    grid_width = kwargs.get(\"grid_width\",15)\n    magnitude  = kwargs.get(\"magnitude\",1)\n    brightness = kwargs.get(\"brightness\",2)\n    contrast = kwargs.get(\"contrast\",2)\n    translate = kwargs.get(\"translate\",0.1)\n    blur_max = kwargs.get(\"blur_max\",2)\n    blur_min = kwargs.get(\"blur_min\",0.1)\n    blur_kernel = kwargs.get(\"blur_kernel\",5)\n                \n\n    # p = Augmentor.Pipeline()\n    # # p.rotate(probability=1,max_left_rotation=max_rotation_left,max_right_rotation=max_rotation_right)\n    # # p.zoom(probability=1,min_factor=min_factor,max_factor=max_factor)\n    # p.random_distortion(grid_width=grid_width,grid_height=grid_height,magnitude=magnitude,probability=1)\n    #\n    #elastic_trans = Distort(probability, grid_width, grid_height, magnitude)\n\n\n    trans_aug = transforms.RandomApply([RandomDistortion(1, grid_width, grid_height, magnitude),\n                                        transforms.ColorJitter(brightness=brightness,contrast=contrast),\n                                        transforms.RandomAffine(translate=(translate,translate),scale=(min_factor,max_factor),\n                                                                degrees=(max_rotation_left,max_rotation_right)),\n                                        #transforms.ToTensor(),\n                                        transforms.GaussianBlur(kernel_size=blur_kernel, sigma=(blur_min, blur_max)),\n                                        SaltPepperNoise(mean=0,std=1/784)\n                                                        ],p=probability)\n\n    trans_train = transforms.Compose([\n                                    transforms.ToPILImage(),\n                                    trans_aug,\n                                    transforms.ToTensor()\n                                    #MapToInterval()\n                                    ])\n    trans_test = transforms.Compose([transforms.ToPILImage(),\n                                     transforms.ToTensor()\n                                     ])\n\n\n    # trans = transforms.Compose([\n    #     p.torch_transform(),\n    #     transforms.ColorJitter(brightness=brightness,contrast=contrast),\n    #\n    #     transforms.ToTensor()\n    # ])\n#     if write_params:\n#         out_dict = dict(probability=probability,\n#                         max_rotation_right=max_rotation_right,\n#                         max_rotation_left=max_rotation_left,\n#                         max_factor = max_factor,\n#                         min_factor = min_factor,\n#                         grid_height = grid_height,\n#                         grid_width = grid_width,\n#                         magnitude = magnitude\n#                         )\n#         with open(os.path.join(log_dir,\"transform_params.txt\"),'w') as f:\n#             # for op in p.operations:\n#             #     f.write(str(op.__class__)+ '\\n')\n#             for op in trans_aug.transforms:\n#                 f.write(str(op.__repr__) + '\\n')\n            # for key,val in out_dict.items():\n            #     f.write('%s:%s\\n' % (key, val))\n\n\n\n    return trans_train,trans_test\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:21.933118Z","iopub.execute_input":"2022-10-23T21:20:21.934004Z","iopub.status.idle":"2022-10-23T21:20:22.217211Z","shell.execute_reply.started":"2022-10-23T21:20:21.933964Z","shell.execute_reply":"2022-10-23T21:20:22.216050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n\ndef make_train_test_log_dir(model,log_loc='./logs'):\n    out_dir   = make_log_dir(model,log_loc)\n    train_loc = os.path.join(out_dir,'train')\n    #if not os.path.isdir(train_loc): os.makedirs(train_loc)\n    val_loc   = os.path.join(out_dir,'test')\n    #if not os.path.isdir(val_loc): os.makedirs(val_loc)\n    return train_loc,val_loc,out_dir\n\n\n\ndef make_log_dir(model,log_loc = './logs'):\n    \"\"\"\n    Uses the model name and creates a numbered folder withing model name\n        example\n    from Architectures import simple_model\n    model = simple_model()\n    x = make_lod_dir(model)\n    print(x)\n    './logs/simple_model/1'\n    :param model:\n    :param log_loc:\n    :return:\n    \"\"\"\n    log_directory = os.path.join(log_loc,model._get_name())\n    #if not os.path.isdir(log_directory): os.makedirs(log_directory)\n    return update_dir(log_directory)\n\n\ndef update_dir(nameInit):\n    \"\"\"\n    Functions that creates numbered folders within directory nameInit. If a folder with name x where x is a number\n    exists, the function creates through i until x+i is no more a folder name. Then creates the folder with name\n    nameInit/[x+i]\n    :param nameInit: path within which the numbered folder needs to be located\n    :return: path of the newly created folder\n    example\n    nameInit = ./\n    ls ./ returns folder1 1 2 3\n    x = update_dir('./)\n    print(x)\n    ./4\n    \"\"\"\n    output_directory = nameInit\n    if not os.path.exists(output_directory):\n        output_directory = os.path.join(output_directory,'1')\n    else:\n        highest_num = 0\n        for f in os.listdir(output_directory):\n            if os.path.exists(os.path.join(output_directory, f)):\n                file_name = os.path.splitext(f)[0]\n                try:\n                    file_num = int(file_name)\n                    if file_num > highest_num:\n                        highest_num = file_num\n                except ValueError:\n                    print('The file name \"%s\" is not an integer. Skipping' % file_name)\n\n        output_directory = os.path.join(output_directory, str(highest_num + 1))\n    #os.makedirs(output_directory)\n    return output_directory\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:22.218642Z","iopub.execute_input":"2022-10-23T21:20:22.218990Z","iopub.status.idle":"2022-10-23T21:20:22.230724Z","shell.execute_reply.started":"2022-10-23T21:20:22.218955Z","shell.execute_reply":"2022-10-23T21:20:22.229620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create trainer class","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch, os, shutil\nfrom pathlib import Path\nimport torch.optim as optim\n\nimport torch.nn as nn\n\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.tensorboard import SummaryWriter\n\nfrom sklearn.metrics import confusion_matrix\n#from util import make_train_test_log_dir, make_log_dir\n#from custom_transforms import create_train_test_transform\nimport seaborn as sns\nimport pandas as pd\n\n\n\nclass Trainer(object):\n    def __init__(self,model,optimizer='Adam',device='cuda',loss_fn='cross-entropy',transform=False,train_file =\n    \"Kannada-MNIST/train.csv\",val_file = \"Kannada-MNIST/Dig-MNIST.csv\",log_dir='./logs',create_save_loc=True,kwargs={}):\n        self.model = model\n        self.kwargs = kwargs\n        if create_save_loc:\n            self._create_save_locs(log_dir)\n        else:\n            self._create_train_test_locs(log_dir)\n        self._get_optimizer(optimizer)\n        self._get_scheduler()\n        if device=='cuda':\n            self.device = torch.device(\"cuda:0\" if torch.cuda.is_available() else 'cpu')\n\n        else:\n            self.device = torch.device('cpu')\n        self.model.to(self.device)\n\n        self._get_loss_function(loss_fn)\n        if transform:\n            self._init_transform(kwargs)\n        else:\n            self.transform = None\n        self._init_data_pipeline(train_file,val_file)\n\n\n\n        self._init_writers()\n\n\n    def _init_transform(self,kwargs):\n        #self.transform = create_transform(log_dir=self.model_outdir,write_params=True,kwargs=kwargs)\n        self.train_trans,self.val_trans = create_train_test_transform(log_dir=self.model_outdir,write_params=True,\n                                                             kwargs=kwargs)\n\n\n    def _get_optimizer(self,optimizer):\n        l_rate = self.kwargs.get(\"l_rate\",0.01)\n        weight_decay = self.kwargs.get(\"weight_decay\",0.0)\n        if optimizer.lower() == 'adam':\n            self.optimizer = optim.Adam(self.model.parameters(),lr=l_rate,weight_decay=weight_decay)\n        elif optimizer.lower()=='momentum':\n            momentum     = self.kwargs.get(\"momentum\",0.0)\n\n            dampening = self.kwargs.get(\"dampening\",0.0)\n            nesterov = bool(self.kwargs.get(\"nesterov\",0))\n\n            self.optimizer = optim.SGD(self.model.parameters(),lr=l_rate,momentum=momentum,weight_decay=weight_decay,\n                                  dampening=dampening,nesterov=nesterov)\n        else:\n            print('optimizer class with name '+optimizer+' not implemented yet')\n            self.optimizer=None\n\n    def _get_scheduler(self):\n        lr_gamma = self.kwargs.get(\"lr_gamma\",1.0)\n        if self.optimizer is not None:\n            self.scheduler = optim.lr_scheduler.ExponentialLR(self.optimizer,gamma=lr_gamma)\n\n    def _get_loss_function(self,loss_fn):\n        \"\"\"\n        Returns the loss function to be used during training.\n        Currently only returns the crossentropy loss, but can be extended to take a custom loss_fn or a different one\n        :param loss_fn:\n        :return:\n        \"\"\"\n        self.loss_fn= nn.CrossEntropyLoss()\n\n    def _init_data_pipeline(self,train_file,test_file):\n        # if split_train_file:\n        if os.path.isfile(train_file):\n            df = pd.read_csv(train_file)\n        train_csv, val_csv = train_test_split(df.iloc[:,0:], test_size=0.2)\n        train_csv.reset_index(drop=True,inplace=True)\n        val_csv.reset_index(drop=True,inplace=True)\n\n\n\n        train_dataset = MnistDataset(train_csv,im_size=(28,28),transforms=self.train_trans)\n        \n        test_dataset = MnistDataset(val_csv,im_size=(28,28),transforms=self.val_trans)\n\n\n#         train_dataset = MnistDataset(train_file,im_size=(28,28),transforms=self.train_trans)\n#         test_dataset = MnistDataset(test_file,im_size=(28,28),transforms=self.val_trans)\n\n        batch_size = self.kwargs.get(\"batch_size\",100)\n        self.train_loader = DataLoader(train_dataset,batch_size=batch_size,shuffle=True)\n        self.test_loader = DataLoader(test_dataset,batch_size=batch_size,shuffle=False)\n\n    def _create_save_locs(self,log_dir):\n        self.train_log_dir,self.test_log_dir,self.model_outdir = make_train_test_log_dir(model=self.model,\n                                                                                        log_loc=log_dir)\n\n    def _create_train_test_locs(self,log_dir):\n        self.model_outdir = log_dir\n        self.train_log_dir = os.path.join(log_dir,'train')\n        if not os.path.isdir(self.train_log_dir): os.makedirs(self.train_log_dir)\n        self.test_log_dir = os.path.join(log_dir,'test')\n        if not os.path.isdir(self.test_log_dir): os.makedirs(self.test_log_dir)\n\n    def _init_writers(self):\n        self.train_writer = None#SummaryWriter(log_dir=self.train_log_dir)\n        self.test_writer = None#SummaryWriter(log_dir=self.test_log_dir)\n\n\n    def _train_block(self,global_step):\n        global_step = train_block(device=self.device,model=self.model,optimizer=self.optimizer,scheduler=self.scheduler,\n                    data_loader=self.train_loader,writer=self.train_writer,global_step=global_step,loss_fn=self.loss_fn)\n        return global_step\n    def _eval_block(self,global_step):\n        eval_block(device=self.device,model=self.model,data_loader=self.test_loader,writer=self.test_writer,\n                   global_step=global_step,\n                   loss_fn=self.loss_fn)\n\n    def _save_checkpt(self,epoch):\n        save_checkpt(model=self.model,optimizer=self.optimizer,scheduler=self.scheduler,epoch=epoch,\n                     out_dir=self.model_outdir)\n\n    def train(self,epochs=2,resume_training=False):\n\n\n        global_step = 0\n        self._create_log_files()\n        for epoch in range(epochs):\n            print(\"======epoch={:}\".format(epoch))\n            global_step = self._train_block(global_step=global_step)\n            self._eval_block(global_step=global_step)\n#             self._save_checkpt(epoch=epoch+1)\n\n\n    def _create_log_files(self):\n        pass\n\n#         with open(os.path.join(self.model_outdir,'training_setup.txt'),'w') as f:\n#             for key,val in self.kwargs.items():\n#                 f.write('%s:%s\\n' % (key,val))\n\n\n\n\n\n\n\n\n\n\ndef train_block(device,model,optimizer,scheduler,data_loader,writer,global_step,loss_fn=nn.CrossEntropyLoss()):\n    #training_loss     = 0.0\n    #training_accuracy = 0.0\n    #running_loss = 0.0\n\n    model.train()\n    scheduler.step()\n    for i,data in enumerate(data_loader,0):\n        global_step+=1\n        inputs,labels = data[0].to(device,dtype=torch.float),data[1].to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = loss_fn(outputs,labels)\n        loss.backward()\n        optimizer.step()\n\n\n        #training_loss += loss.item()\n\n        #if global_step % 300 == 99:  # print every 2000 mini-batches\n            # pred_score,pred_class = outputs.max(dim=1)\n            # training_accuracy = (pred_class == labels).sum().item()\n            # training_accuracy /= inputs.shape[0]\n            # training_loss = loss.item()\n            # print(f'[{epoch + 1}, {global_step:5d}] loss: {training_loss :.3f} accuracy: {training_accuracy}')\n\n            # train_writer.add_scalar('accuracy',training_accuracy,global_step=global_step)\n            # train_writer.add_scalar('loss',training_loss,global_step=global_step)\n            #add_to_writer(writer,loss_fn,inputs,outputs,labels,iter=global_step)\n\n    return global_step\n\n\n\ndef eval_block(device,model,data_loader,writer,global_step,loss_fn=nn.CrossEntropyLoss()):\n    #training_loss     = 0.0\n    #training_accuracy = 0.0\n    #running_loss = 0.0\n\n    eval_loss     = 0.0\n    eval_accuracy = 0.0\n    model.eval()\n    with torch.no_grad():\n        input_list = []\n        label_list = []\n        output_list = []\n        for i,data in enumerate(data_loader,0):\n\n            inputs,labels = data[0].to(device,dtype=torch.float),data[1].to(device)\n\n            outputs =model(inputs)\n            input_list +=[inputs]\n            output_list += [outputs]\n            label_list += [labels]\n\n\n            #loss = loss_fn(outputs,labels)\n\n            #pred_score,pred_class = outputs.max(dim=1)\n            # eval_accuracy += (pred_class == labels).sum().item()\n            # eval_loss     += loss.item()*inputs.shape[0]\n\n        merged_inputs  = torch.cat(input_list,axis=0)\n        merged_outputs = torch.cat(output_list,axis=0)\n        merged_labels  = torch.cat(label_list,axis=0)\n        #add_to_writer(writer,loss_fn,merged_inputs,merged_outputs,merged_labels,iter=global_step)\n\n\n\n\n\ndef make_prediction_fig(inputs,pred_class,labels,iter):\n\n    fig = plt.figure(figsize=(12, 16))\n    for idx in np.arange(16):\n        ax = fig.add_subplot(4, 4, idx+1, xticks=[], yticks=[])\n        ax.imshow(inputs[idx,0,:,:].cpu(),cmap='Greys_r')\n        ax.set_title(\"Predicted Class {:},\\n true label={:}\".format(pred_class[idx].cpu(),labels[idx].cpu()))\n    plt.tight_layout()\n        # ax.set_title(\"{0}, {1:.1f}%\\n(label: {2})\".format(\n\n        #     classes[preds[idx]],\n        #     probs[idx] * 100.0,\n        #     classes[labels[idx]]),\n        #             color=(\"green\" if preds[idx]==labels[idx].item() else \"red\"))\n    return fig\n\n\ndef make_confusion_matrix(inputs,pred_class,labels,iter):\n    #TODO: THIS IS NOT DOING WHAT i WANT IT TO DO\n    print(\"input to confusion matrix\")\n    print(labels.shape)\n    print(pred_class.shape)\n    cf_matrix = confusion_matrix(labels.cpu().detach().numpy(),pred_class.cpu().detach().numpy())\n    df_cm = pd.DataFrame(cf_matrix / np.sum(cf_matrix,axis=0,keepdims=True),index=[i for i in range(10)],\n                         columns=[i for i in range(10)])\n\n    # Plot the dataframe using seaborn\n    fig = plt.figure(figsize=(12,7))\n    sns.heatmap(df_cm,annot=True)\n    plt.title(\"Neural Network Confusion Matrix step{:}\".format(iter),fontsize=20)\n    return fig\n\ndef add_to_writer(writer,loss_fn,inputs,outputs,labels,iter):\n    \"\"\"\n    Function that adds performance metrics ro a tensorboard summary writer at training glbal step=iter\n    also prints the accuracy and loss metrics\n    :param writer:\n    :param loss_fn:\n    :param inputs:\n    :param outputs:\n    :param labels:\n    :param iter:\n    :return:\n    \"\"\"\n    pred_score,pred_class = outputs.max(dim=1)\n    accuracy = (pred_class == labels).sum().item()\n    accuracy /= inputs.shape[0]\n    loss = loss_fn(outputs,labels)\n    loss_val = loss.item()\n    print(f'global_step{iter:5d} loss: {loss_val :.3f} accuracy: {accuracy}')\n    writer.add_scalar('accuracy',accuracy,global_step=iter)\n    writer.add_scalar('loss',loss_val,global_step=iter)\n    fig = make_prediction_fig(inputs,pred_class,labels,iter)\n    writer.add_figure('predictions',fig,global_step=iter)\n    # fig = make_confusion_matrix(inputs,pred_class,labels,iter)\n\n    # writer.add_figure('Confusion matrix',fig,global_step=iter)\n\n\ndef save_checkpt(model,optimizer,scheduler,epoch,out_dir,max_keep=10,delete_prev=True):\n    checkpoints = [f for f in os.listdir(out_dir) if 'checkpoint' in f]\n    if len(checkpoints)>max_keep:\n        for f in checkpoints:\n            os.remove(os.path.join(out_dir,f))\n\n    file_path = os.path.join(out_dir,'checkpoint_epoch_{:}.pt'.format(epoch))\n    torch.save({'epoch'     : epoch,\n                'net'       : model.state_dict(),\n                'optimizer' : optimizer.state_dict(),\n                'scheduler' : scheduler.state_dict()},file_path\n               )\n\n\n\ndef load_checkpt(model,optimizer,scheduler,epoch,out_dir):\n    file_path = os.path.join(out_dir,'checkpoint_epoch_{:}.pt'.format(epoch))\n    print('loading '+file_path)\n    checkpoint = torch.load(file_path)\n    model.load_state_dict(checkpoint['net'])\n    optimizer.load_state_dict(checkpoint['optimizer'])\n    scheduler.load_state_dict(checkpoint['scheduler'])\n    return model,optimizer,scheduler\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:22.232678Z","iopub.execute_input":"2022-10-23T21:20:22.233350Z","iopub.status.idle":"2022-10-23T21:20:23.210118Z","shell.execute_reply.started":"2022-10-23T21:20:22.233312Z","shell.execute_reply":"2022-10-23T21:20:23.209056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.cuda\nimport numpy as np\nimport csv\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\n\nclass mnist_model_pool(nn.Module):\n    def __init__(self,init_width=4,dropout_rate=0.4):\n        super(mnist_model_pool,self).__init__()\n\n        self.init_width   = init_width\n        self.dropout_rate = dropout_rate\n\n        self.init_layers()\n\n\n    def init_layers(self):\n        #input size (28,28)\n        self.conv1_1 = nn.Conv2d(in_channels=1,out_channels=self.init_width,kernel_size=5,stride=1\n                                 ,padding=2)\n        #num weights= 3*4*1=12\n        self.conv1_2 = nn.Conv2d(in_channels=self.init_width,out_channels=self.init_width,kernel_size=5,stride=1,\n                                 padding=2)\n\n        self.conv11_bn = nn.BatchNorm2d(num_features=self.init_width)\n        self.conv12_bn = nn.BatchNorm2d(num_features=self.init_width)\n        \n        #num weights= 3*4*4=48, num biases = 4\n        #size (14,14,4)\n\n        #self.batch1 = nn.BatchNorm2d(num_features=self.init_width)\n        #input size is halved (14,14)\n        self.conv2_1 = nn.Conv2d(in_channels=self.init_width,out_channels=self.init_width*2,kernel_size=5,stride=1,\n                                 padding=2)\n        #num weights= 3*4*8=96 num biases = 8\n        self.conv2_2 = nn.Conv2d(in_channels=self.init_width*2,out_channels=self.init_width*2,kernel_size=5,stride=1,\n                                 padding=2)\n\n        self.conv21_bn = nn.BatchNorm2d(num_features=self.init_width*2)\n        self.conv22_bn = nn.BatchNorm2d(num_features=self.init_width*2)\n        #num weights= 3*8*8=192 num biases = 8\n        #size (14,14,8)\n        #input size is halved (7,7)\n        self.conv3_1 = nn.Conv2d(in_channels=self.init_width*2,out_channels=self.init_width*4,kernel_size=3,stride=1,\n                                 padding=1)\n        #num weights= 3*8*16=384 num biases = 16\n        self.conv3_2 = nn.Conv2d(in_channels=self.init_width * 4,out_channels=self.init_width * 4,kernel_size=3,\n                                 stride=1,\n                                 padding=1)\n\n        self.conv31_bn = nn.BatchNorm2d(num_features=self.init_width*4)\n        self.conv32_bn = nn.BatchNorm2d(num_features=self.init_width*4)\n\n\n        self.conv4_1 = nn.Conv2d(in_channels=self.init_width*4,out_channels=self.init_width*8,kernel_size=3,stride=1,\n                                 padding=1)\n        #num weights= 3*8*16=384 num biases = 16\n        self.conv4_2 = nn.Conv2d(in_channels=self.init_width * 8,out_channels=self.init_width * 8,kernel_size=3,\n                                 stride=1,\n                                 padding=1)\n        self.conv41_bn = nn.BatchNorm2d(num_features=self.init_width*8)\n        self.conv42_bn = nn.BatchNorm2d(num_features=self.init_width*8)\n        #size (7,7,16)\n        #input size is halved (4,4)\n        # self.conv4_1 = nn.Conv2d(in_channels=self.init_width * 4,out_channels=self.init_width*8 ,kernel_size=3,\n        #                          stride=2,\n        #                          padding=1)\n        # self.conv4_2 = nn.Conv2d(in_channels=self.init_width * 8,out_channels=self.init_width * 4,kernel_size=3,\n        #                          stride=1,\n        #                          padding=1)\n        #size (4,4,16)\n        self.fc1  = nn.Linear(in_features=self.init_width * 4 * 4*8,out_features=self.init_width*4*4)\n        self.fc1_bn = nn.BatchNorm1d(num_features=self.init_width*4*4)\n        self.fc2  = nn.Linear(in_features=self.init_width * 4 * 4 ,out_features=self.init_width * 4 )\n        self.fc2_bn = nn.BatchNorm1d(num_features=self.init_width * 4 )\n        self.out  = nn.Linear(in_features=self.init_width * 4,out_features=10)\n\n        self.dropout = nn.Dropout(self.dropout_rate)\n        #self.fc5  = nn.Linear(in_features=)\n        #size (7,7,1)\n\n    def forward(self,z):\n        z = F.leaky_relu(self.conv11_bn(self.conv1_1(z)),negative_slope=0.1)\n        z = F.leaky_relu(self.conv12_bn(self.conv1_2(z)),negative_slope=0.1)\n\n        z = F.max_pool2d(z,kernel_size=2)\n        z = F.leaky_relu(self.conv21_bn(self.conv2_1(z)),negative_slope=0.1)\n        z = F.leaky_relu(self.conv22_bn(self.conv2_2(z)),negative_slope=0.1)\n\n        z = F.max_pool2d(z,kernel_size=2)\n        z = F.leaky_relu(self.conv31_bn(self.conv3_1(z)),negative_slope=0.1)\n        z = F.leaky_relu(self.conv32_bn(self.conv3_2(z)),negative_slope=0.1)\n\n        z = F.max_pool2d(z,kernel_size=2,padding=1)\n        z = F.leaky_relu(self.conv41_bn(self.conv4_1(z)),negative_slope=0.1)\n        z = F.leaky_relu(self.conv42_bn(self.conv4_2(z)),negative_slope=0.1)\n        #\n\n        # z = F.relu(self.conv4_1(z))\n        # z = F.relu(self.conv4_2(z))\n        #\n        z = torch.flatten(z,1)\n        z = F.relu(self.fc1_bn(self.fc1(z)))\n        z = self.dropout(z)\n        z = F.relu(self.fc2_bn(self.fc2(z)))\n        #z = self.dropout(z)\n        z = self.out(z)\n        return z\n\n\n\ndef count_nn_params(model):\n    \"\"\"\n    counts the number of trainable weights+biases in the nn\n    :param model:\n    :return:\n    \"\"\"\n    num_params = 0\n    for param in list(model.parameters()):\n        n = 1\n        for s in list(param.size()):\n            n = n*s\n        num_params += n\n    return num_params\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:23.211544Z","iopub.execute_input":"2022-10-23T21:20:23.211944Z","iopub.status.idle":"2022-10-23T21:20:23.239782Z","shell.execute_reply.started":"2022-10-23T21:20:23.211900Z","shell.execute_reply":"2022-10-23T21:20:23.238640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\nimport numpy as np\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom torchvision import transforms\n#from Augmentor.Operations import Distort\n\nlabel_dict = {'0' : 'shunya',\n              '1' :'omdu',\n              '2' :'eradu',\n              '3' :'muru',\n              '4' :'nalku',\n              '5' :'aidu',\n              '6' :'aru',\n              '7' :'elu',\n              '8' :'emtu',\n              '9' :'ombattu',\n              }\n\n# def load_csv(filePath):\n#     \"\"\"\n#     This is a generic method to read csvs by bypassing pandas\n#     :param filePath:string, abosolute file path to the csv file to be read\n#     :return:  dictionary with keys being the column headers and items being the column vectors\n#     \"\"\"\n#     my_dict = {}\n#     with open(filePath,mode='r') as infile:\n#         reader = csv.reader(infile)\n#         row_count = 0\n#         for row in reader:\n#             if row_count==0:\n#                 keys = row\n#\n#             else:\n#                 for key,it in zip(keys,row):\n#                     if row_count==1:\n#                         out_arr = np.expand_dims(np.array(it),axis=0)\n#                     else:\n#                         out_arr_ = np.expand_dims(np.array(it),axis=0)\n#                         out_arr  = np.concatenate([out_arr,out_arr_],axis=0)\n#                     # if row_count==1:\n#                     #     my_dict[key] = [it]\n#                     # else:\n#                     #     my_dict[key] += [it]\n#             row_count+=1\n#\n#\n#     return out_arr\n\n\nclass MnistDataset(Dataset):\n    def __init__(self,csvfile,im_size,transforms=None):\n        super(MnistDataset,self).__init__()\n        if type(csvfile) is str:\n            self.dataframe = pd.read_csv(csvfile)\n        elif csvfile.__module__ == 'pandas.core.frame':\n            # self.csvfile    = csvfile\n            self.dataframe = csvfile\n\n        self.im_size    = im_size\n        self.transforms = transforms\n\n\n\n    def __len__(self):\n        return len(self.dataframe)\n\n\n    def __getitem__(self,i):\n        row = self.dataframe.iloc[i,:]\n        im   = np.array(row[1:]).astype(np.uint8).reshape(self.im_size[0],self.im_size[1],1)\n        #im   = im.astype(np.float32)\n        #image = (im - np.min(im)) / (np.max(im) - np.min(im))\n        if self.transforms:\n            im = self.transforms(im)\n        lab   = row[0]\n        return im, lab\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:23.241810Z","iopub.execute_input":"2022-10-23T21:20:23.242244Z","iopub.status.idle":"2022-10-23T21:20:23.257481Z","shell.execute_reply.started":"2022-10-23T21:20:23.242205Z","shell.execute_reply":"2022-10-23T21:20:23.256413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DATAPATH=\"../input/Kannada-MNIST/\"\nTRAIN_FILE=\"../input/Kannada-MNIST/train.csv\"\nTEST_FILE=\"../input/Kannada-MNIST/test.csv\"\nSAMPLE_FILE =\"../input/Kannada-MNIST/sample_submission.csv\"\nDIG_FILE=\"../input/Kannada-MNIST/Dig-MNIST.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:23.261362Z","iopub.execute_input":"2022-10-23T21:20:23.261819Z","iopub.status.idle":"2022-10-23T21:20:23.271479Z","shell.execute_reply.started":"2022-10-23T21:20:23.261782Z","shell.execute_reply":"2022-10-23T21:20:23.270344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntest_path = \"/home/adwaye/PycharmProjects/kannadaMnist/Kannada-MNIST/test.csv\"\nsample_submission = \"/home/adwaye/PycharmProjects/kannadaMnist/Kannada-MNIST/sample_submission.csv\"\n\nmodel      = mnist_model_pool(init_width=init_width)\nlog_loc = './logs'\nmy_trainer = Trainer(model,kwargs=opt_kwargs,log_dir=DATAPATH,create_save_loc=True,transform=True,train_file=TRAIN_FILE)\nmy_trainer.train(epochs=epochs)\n\n\n\nsubmission_dataset = MnistDataset(csvfile=SAMPLE_FILE,im_size=(28,28))\nsubmission_loader  = DataLoader(submission_dataset,batch_size=100)\n\n\n# Make sure the model is in eval mode!\n\n# submission_file = pd.read_csv(sample_submission)\n# submission_file['label'] = predictions.cpu().numpy()\n# submission_file.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:20:23.273236Z","iopub.execute_input":"2022-10-23T21:20:23.273672Z","iopub.status.idle":"2022-10-23T21:21:49.136462Z","shell.execute_reply.started":"2022-10-23T21:20:23.273636Z","shell.execute_reply":"2022-10-23T21:21:49.135506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel.to(my_trainer.device)\nmodel.eval()\ntrain_trans,test_trans = create_train_test_transform(log_dir='./')\nsubmission_dataset = MnistDataset(csvfile=TEST_FILE,im_size=(28,28),transforms=test_trans)\n\nsubmission_loader  = DataLoader(submission_dataset,batch_size=100)\npredictions = torch.LongTensor().to(my_trainer.device)\nfor i,data in enumerate(submission_loader):\n    if i ==0:\n        print(data[0].size())\n    preds = model(data[0].to(my_trainer.device,dtype=torch.float))\n\n    predictions = torch.cat((predictions, preds.argmax(dim=1)), dim=0)\nsubmission_file = pd.read_csv(SAMPLE_FILE)\nsubmission_file['label'] = predictions.cpu().numpy()\nsubmission_file.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:21:49.138063Z","iopub.execute_input":"2022-10-23T21:21:49.138443Z","iopub.status.idle":"2022-10-23T21:21:51.742185Z","shell.execute_reply.started":"2022-10-23T21:21:49.138406Z","shell.execute_reply":"2022-10-23T21:21:51.741226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_file\n","metadata":{"execution":{"iopub.status.busy":"2022-10-23T21:21:51.743623Z","iopub.execute_input":"2022-10-23T21:21:51.744099Z","iopub.status.idle":"2022-10-23T21:21:51.759221Z","shell.execute_reply.started":"2022-10-23T21:21:51.744062Z","shell.execute_reply":"2022-10-23T21:21:51.758190Z"},"trusted":true},"execution_count":null,"outputs":[]}]}