{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"load datasets from kaggle \n\nhttps://www.analyticsvidhya.com/blog/2021/06/how-to-load-kaggle-datasets-directly-into-google-colab/\n\nbefore first line you need to import kaggle.json into filesystem\n\n","metadata":{"id":"XAjWKGKNR2he"}},{"cell_type":"code","source":"# google colab\n#! mkdir ~/.kaggle\n#! cp kaggle.json ~/.kaggle/\n#! chmod 600 ~/.kaggle/kaggle.json\n#! kaggle competitions download cifar-10\n#!unzip cifar-10.zip","metadata":{"id":"z6_0bpIWR9Ff","execution":{"iopub.status.busy":"2022-03-25T20:45:39.901395Z","iopub.status.idle":"2022-03-25T20:45:39.902105Z","shell.execute_reply.started":"2022-03-25T20:45:39.901837Z","shell.execute_reply":"2022-03-25T20:45:39.901866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#kaggle notebook\n!pip install torchsummary\n!pip install --upgrade torch torchvision","metadata":{"id":"5o0oS8hp411o","execution":{"iopub.status.busy":"2022-04-04T16:36:56.432784Z","iopub.execute_input":"2022-04-04T16:36:56.433097Z","iopub.status.idle":"2022-04-04T16:38:14.536741Z","shell.execute_reply.started":"2022-04-04T16:36:56.433018Z","shell.execute_reply":"2022-04-04T16:38:14.535939Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr","metadata":{"id":"rC2kza8WS2XV","execution":{"iopub.status.busy":"2022-04-04T16:39:52.779010Z","iopub.execute_input":"2022-04-04T16:39:52.780032Z","iopub.status.idle":"2022-04-04T16:40:03.413667Z","shell.execute_reply.started":"2022-04-04T16:39:52.779988Z","shell.execute_reply":"2022-04-04T16:40:03.412881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python -m py7zr x ./train.7z # colab \n!python -m py7zr x ../input/cifar-10/train.7z #kaggle notebook","metadata":{"id":"bvbhfFaOSXXR","execution":{"iopub.status.busy":"2022-04-04T16:40:08.467545Z","iopub.execute_input":"2022-04-04T16:40:08.467852Z","iopub.status.idle":"2022-04-04T16:40:50.827198Z","shell.execute_reply.started":"2022-04-04T16:40:08.467820Z","shell.execute_reply":"2022-04-04T16:40:50.826258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python -m py7zr x ./test.7z\n!python -m py7zr x ../input/cifar-10/test.7z #kaggle notebook","metadata":{"id":"dtgU1XeXTmm-","execution":{"iopub.status.busy":"2022-04-04T16:55:31.675973Z","iopub.execute_input":"2022-04-04T16:55:31.676623Z","iopub.status.idle":"2022-04-04T17:09:22.473543Z","shell.execute_reply.started":"2022-04-04T16:55:31.676585Z","shell.execute_reply":"2022-04-04T17:09:22.472607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#imports\nimport math\nimport random\nimport copy\nimport os\n\nimport pandas as pd\nimport numpy as np\n\nimport torch\nfrom torchvision.io import read_image\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision.transforms as T\nfrom torchsummary import summary\nfrom torchvision import models\n\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\n\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\n\nfrom tqdm import tqdm\n","metadata":{"id":"g9eilYG0Xj1W","execution":{"iopub.status.busy":"2022-04-04T16:42:06.819534Z","iopub.execute_input":"2022-04-04T16:42:06.819815Z","iopub.status.idle":"2022-04-04T16:42:08.466737Z","shell.execute_reply.started":"2022-04-04T16:42:06.819780Z","shell.execute_reply":"2022-04-04T16:42:08.466002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#variable\nIMG_SIZE = 32","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:42:10.511653Z","iopub.execute_input":"2022-04-04T16:42:10.512393Z","iopub.status.idle":"2022-04-04T16:42:10.518360Z","shell.execute_reply.started":"2022-04-04T16:42:10.512356Z","shell.execute_reply":"2022-04-04T16:42:10.517451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hyp = {\n    #regularization hyperparameters\n    \"dropout_rate\" : 0.2,\n    \"data_augmentation_rate\" : 0.06, # probabilty of any transformation = 1 - (1 - data_augmentation_rate)**(Number of transformation)\n    \"data_augmentation_rate_mix\" : 0.03,\n\n\n\n    #leraning hyperparameters\n    \"batch_size\" : 256,\n    \"lerning_rate\" :  1e-3,\n    \"shuffle\" : True\n}\n","metadata":{"id":"XAxHMwVDa3Ru","execution":{"iopub.status.busy":"2022-04-04T16:42:11.722648Z","iopub.execute_input":"2022-04-04T16:42:11.722920Z","iopub.status.idle":"2022-04-04T16:42:11.727704Z","shell.execute_reply.started":"2022-04-04T16:42:11.722890Z","shell.execute_reply":"2022-04-04T16:42:11.726644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomImageDataset(Dataset):\n    def __init__(self,  img_dir,annotations_file = None,transform = None):\n      self.img_labels = None\n      self.le = preprocessing.LabelEncoder()\n      if  annotations_file is not None:\n        self.img_labels = pd.read_csv(annotations_file)\n        self.img_labels['index_label'] = self.le.fit_transform(self.img_labels.iloc[:,1])\n      self.img_dir = img_dir\n      self.transform = transform\n\n    def __len__(self):\n      if self.img_labels is None:\n        return len(os.listdir(self.img_dir))\n      return len(self.img_labels)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, str(idx + 1) + '.png')\n        image = read_image(img_path)\n        label = F.one_hot(torch.tensor(1),10).float()\n        if  self.img_labels is not None:\n          label = self.img_labels.iloc[idx, 2]\n          label = torch.tensor(int(label))\n          label = F.one_hot(label,10).float()\n\n        # apply transformations\n        if self.transform:\n          for t in self.transform:\n            image = t(image)\n        return image, label\n          \n    def getLabel(self,idx):\n      if  self.img_labels is not None:\n        return self.img_labels.iloc[idx,1]\n      return None\n\n    def encodeLabel(self, idx):\n      if self.img_labels is None:\n        return None\n      return self.le.inverse_transform(idx)","metadata":{"id":"vMJEubPKX4j1","execution":{"iopub.status.busy":"2022-04-04T16:42:13.506416Z","iopub.execute_input":"2022-04-04T16:42:13.507118Z","iopub.status.idle":"2022-04-04T16:42:13.519553Z","shell.execute_reply.started":"2022-04-04T16:42:13.507066Z","shell.execute_reply":"2022-04-04T16:42:13.518759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RandomProbabilityWrapper:\n\n    def __init__(self,probability,transformation):\n        self.transformation = transformation\n        self.probability = probability\n\n\n    def __call__(self, img):\n        if random.random() < self.probability:\n            return self.transformation(img)\n        return img","metadata":{"id":"FiUWDogZa0Je","execution":{"iopub.status.busy":"2022-04-04T16:42:15.687471Z","iopub.execute_input":"2022-04-04T16:42:15.687903Z","iopub.status.idle":"2022-04-04T16:42:15.694056Z","shell.execute_reply.started":"2022-04-04T16:42:15.687858Z","shell.execute_reply":"2022-04-04T16:42:15.693358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Cutmix:\n  def __init__(self,probability, dataset):\n    self.probability = probability\n    self.dataset = dataset\n    \n  def transform(self, batch):\n    indexes = np.where(np.random.rand(len(batch[0])) < self.probability)[0]\n    print(indexes)\n    for i in indexes:\n        rand_index = random.randint(0,len(self.dataset) - 1 )\n        second_image, second_label = self.dataset[rand_index]\n        batch[0][i], batch[1][i] = self(batch[0][i], batch[1][i],second_image, second_label)\n    return batch\n    \n\n\n  def __call__(self, image1, label1, image2, label2, beta_coff = 0.3,debug = False):\n    ratio = np.random.beta(beta_coff, beta_coff, size=None)\n    a = int(math.sqrt(ratio) * IMG_SIZE)\n    x = random.randint(0, IMG_SIZE -1)\n    y = random.randint(0, IMG_SIZE -1)\n    x2 = x + a if x + a < IMG_SIZE else IMG_SIZE\n    y2 = y + a if y + a < IMG_SIZE else IMG_SIZE\n    width = x2 - x\n    height = y2 - y\n    ratio = width * height * 1.0 / (IMG_SIZE ** 2)\n\n    crop2 = T.functional.crop(image2, y, x, height, width) #take only selected part \n    image2 = T.functional.pad(crop2, [x, y, IMG_SIZE - x2, IMG_SIZE - y2]) #set it on image dimenzions black image\n\n    crop1 = T.functional.crop(image1, y, x, height, width)\n    img1 = T.functional.pad(crop1, [x, y, IMG_SIZE - x2, IMG_SIZE - y2])\n\n    image1 = image1 - img1\n    image = image1 + image2\n\n    label = ratio * label2 + (1 - ratio) * label1\n    if debug == True:\n        print(\"a = \" + str(a))\n        print(\"x, y = \" +str(x) + \", \"+str(y))\n        print(\"x2, y2 = \" +str(x2) + \", \"+str(y2))\n        print(\"width, height = \" +str(width) + \", \"+str(height))\n        \n    return image, label","metadata":{"id":"g1_KVaVW3u-F","execution":{"iopub.status.busy":"2022-04-04T16:42:17.077209Z","iopub.execute_input":"2022-04-04T16:42:17.077837Z","iopub.status.idle":"2022-04-04T16:42:17.093455Z","shell.execute_reply.started":"2022-04-04T16:42:17.077794Z","shell.execute_reply":"2022-04-04T16:42:17.091233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataAugmentationLayer(nn.Module):\n    def __init__(self, image_transformation = []):\n        super().__init__()\n        self.image_transformation = image_transformation\n\n    def forward(self, x):\n        if self.training == True:\n            for t in self.image_transformation:\n                x = t(x)          \n        return x    ","metadata":{"id":"UQloQSz3411w","execution":{"iopub.status.busy":"2022-04-04T16:42:19.193081Z","iopub.execute_input":"2022-04-04T16:42:19.193733Z","iopub.status.idle":"2022-04-04T16:42:19.198694Z","shell.execute_reply.started":"2022-04-04T16:42:19.193695Z","shell.execute_reply":"2022-04-04T16:42:19.197963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TransformationLayer(nn.Module):\n    def __init__(self, image_transformation = []):\n        super().__init__()\n        self.image_transformation = image_transformation\n\n    def forward(self, x):\n        for t in self.image_transformation:\n            x = t(x)          \n        return x    ","metadata":{"id":"C8r6_YTDcBeP","execution":{"iopub.status.busy":"2022-04-04T16:42:20.952834Z","iopub.execute_input":"2022-04-04T16:42:20.953305Z","iopub.status.idle":"2022-04-04T16:42:20.958839Z","shell.execute_reply.started":"2022-04-04T16:42:20.953266Z","shell.execute_reply":"2022-04-04T16:42:20.957831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cifar10_dataset = CustomImageDataset(\"./train/\",\"./trainLabels.csv\") # google colab\ncifar10_dataset = CustomImageDataset(\"./train/\",\"../input/cifar-10/trainLabels.csv\") #kaggle notebook","metadata":{"id":"MuyrPbls05lK","execution":{"iopub.status.busy":"2022-04-04T16:42:22.374264Z","iopub.execute_input":"2022-04-04T16:42:22.374779Z","iopub.status.idle":"2022-04-04T16:42:22.425753Z","shell.execute_reply.started":"2022-04-04T16:42:22.374741Z","shell.execute_reply":"2022-04-04T16:42:22.425042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_datasets, val_datasets = train_test_split(cifar10_dataset, test_size=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:42:23.996675Z","iopub.execute_input":"2022-04-04T16:42:23.996929Z","iopub.status.idle":"2022-04-04T16:42:40.833742Z","shell.execute_reply.started":"2022-04-04T16:42:23.996900Z","shell.execute_reply":"2022-04-04T16:42:40.833041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure\nfigure(figsize=(8, 6), dpi=80)\nimgNum = 2\narr_ = cifar10_dataset[imgNum][0]\nplt.imshow(arr_.T)\nplt.show()\nprint(cifar10_dataset[imgNum][1])\nprint(cifar10_dataset.getLabel(imgNum))\n","metadata":{"id":"RN64kIkubbsg","outputId":"e35e47af-7e08-48d2-8469-d3fc37326b27","execution":{"iopub.status.busy":"2022-04-04T16:42:43.847680Z","iopub.execute_input":"2022-04-04T16:42:43.848411Z","iopub.status.idle":"2022-04-04T16:42:44.031166Z","shell.execute_reply.started":"2022-04-04T16:42:43.848370Z","shell.execute_reply":"2022-04-04T16:42:44.030441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rotation = RandomProbabilityWrapper(hyp[\"data_augmentation_rate\"], lambda x : T.functional.rotate(x,int(np.random.choice([90,180,270]))))\nresize_cropper = RandomProbabilityWrapper(hyp[\"data_augmentation_rate\"], lambda x :T.RandomResizedCrop(size=(32, 32))(x))\naffine = RandomProbabilityWrapper(hyp[\"data_augmentation_rate\"], lambda x :T.RandomAffine(degrees=0, translate=(0.1, 0.4), scale=(1,1))(x))\n\ncutmix = Cutmix(hyp[\"data_augmentation_rate_mix\"],cifar10_dataset)\n\n    ","metadata":{"id":"uDRTpLbRdLiE","execution":{"iopub.status.busy":"2022-04-04T16:42:45.937981Z","iopub.execute_input":"2022-04-04T16:42:45.938538Z","iopub.status.idle":"2022-04-04T16:42:45.944479Z","shell.execute_reply.started":"2022-04-04T16:42:45.938497Z","shell.execute_reply":"2022-04-04T16:42:45.943661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataAugumentation = DataAugmentationLayer(image_transformation = [rotation,resize_cropper,affine])","metadata":{"id":"CEcb4Kmn411x","execution":{"iopub.status.busy":"2022-04-04T16:42:47.760907Z","iopub.execute_input":"2022-04-04T16:42:47.761677Z","iopub.status.idle":"2022-04-04T16:42:47.766279Z","shell.execute_reply.started":"2022-04-04T16:42:47.761626Z","shell.execute_reply":"2022-04-04T16:42:47.765445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_datasets, batch_size=hyp[\"batch_size\"], shuffle=hyp[\"shuffle\"])\nval_dataloader = DataLoader(val_datasets, batch_size=hyp[\"batch_size\"], shuffle=hyp[\"shuffle\"])","metadata":{"id":"IBNCWIrFeMUU","execution":{"iopub.status.busy":"2022-04-04T16:42:49.692407Z","iopub.execute_input":"2022-04-04T16:42:49.693028Z","iopub.status.idle":"2022-04-04T16:42:49.698629Z","shell.execute_reply.started":"2022-04-04T16:42:49.692986Z","shell.execute_reply":"2022-04-04T16:42:49.697039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imgNum = random.randint(0,len(cifar10_dataset))\nimgNum1 =random.randint(0,len(cifar10_dataset))\nfigsize=(8, 6)\nfig, (ax1, ax2,ax3) = plt.subplots(1, 3,figsize=(35, 10) )\ni1,l1 = cifar10_dataset[imgNum]\nax1.imshow(i1.int().T)\nax1.set_title(str(l1) + \"--> \" + cifar10_dataset.encodeLabel([torch.argmax(l1)]))\n\ni2,l2 = cifar10_dataset[imgNum1]\nax2.imshow(i2.int().T)\nax2.set_title(str(l2) + \"--> \" + cifar10_dataset.encodeLabel([torch.argmax(l2)]))\n\n\ncut_test = Cutmix(1,cifar10_dataset)\n\nfigure(figsize=(8, 6), dpi=80)\nimage, label = cut_test(i1,l1,i2,l2,debug = True)\nax3.imshow(image.int().T)\nax3.set_title(str(label) + \"--> \" + cifar10_dataset.encodeLabel([torch.argmax(label)]))\n\n","metadata":{"id":"4uomZjkbkw3n","execution":{"iopub.status.busy":"2022-04-04T16:43:01.882728Z","iopub.execute_input":"2022-04-04T16:43:01.883341Z","iopub.status.idle":"2022-04-04T16:43:02.431798Z","shell.execute_reply.started":"2022-04-04T16:43:01.883300Z","shell.execute_reply":"2022-04-04T16:43:02.431143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train(model, train_dataloader, test_dataloader = None, num_epochs=1, lr=1e-3, keepBest = False, mix_trainsformation = None):\n  optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n  array_loss_train = np.empty([0])\n  array_loss_val = np.empty([0])\n  #with wandb.init(project=\"UZOP_LAB\"):\n  #model.cuda(0)\n  best_model_wts = None\n  best_lost = None\n\n  for epoch in range(num_epochs):\n    correct_train = 0\n    model.train()\n    with tqdm(enumerate(train_dataloader), total=len(train_dataloader), desc=f'Training (epoch={epoch}/{num_epochs})') as epoch_progress:\n      for batch_idx, train_batch in epoch_progress:\n        if mix_trainsformation is not None:\n            for t in mix_trainsformation:\n                train_batch = t.transform(train_batch)\n                \n        x, y = train_batch      \n        x = x.float().cuda()\n        y = y.cuda()\n        logits = model(x)\n        value, predicted = torch.max(logits, 1)\n        loss = F.cross_entropy(logits, y)\n        epoch_correct = (predicted==torch.argmax(y, dim=1)).sum().item()\n        correct_train += epoch_correct\n        epoch_progress.set_postfix({'loss': 1 - epoch_correct / hyp[\"batch_size\"]})  \n\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n    train_loss = 1 - (correct_train / len(train_dataloader.dataset))\n    array_loss_train = np.append(array_loss_train,train_loss)\n    print(\"Train loss \" + str(train_loss))\n\n    if test_dataloader is not None: \n      with torch.no_grad():\n        model.eval()\n        correct_val = 0\n        for test_batch in tqdm(test_dataloader, desc=\"Testing\"):\n          x, y = test_batch\n          x = x.float().cuda()\n          y = y.cuda()\n\n          logits = model(x)\n          value, predicted = torch.max(logits, 1)\n          loss = F.cross_entropy(logits, y)\n          epoch_correct = (predicted==torch.argmax(y, dim=1)).sum().item()\n          correct_val += epoch_correct\n\n        val_loss = 1 - (correct_val / len(test_dataloader.dataset))\n        array_loss_val = np.append(array_loss_val,val_loss)\n        print(\"Val loss \" + str(val_loss))\n\n      if best_lost is None or best_lost > val_loss:\n            best_lost = val_loss\n            best_model_wts = copy.deepcopy(model.state_dict())\n\n  if test_dataloader is not None: \n    if keepBest:\n        model.load_state_dict(best_model_wts)\n      \n    model.eval()\n    \n    with torch.no_grad():\n      test_loss = []\n      correct = 0\n      count = 0\n\n\n      for test_batch in tqdm(test_dataloader, desc=\"Testing\"):\n        x, y = test_batch\n        x = x.float().cuda()\n        y = y.cuda()\n        logits = model(x)\n        loss = F.cross_entropy(logits, y)\n        test_loss.append(loss)\n        correct += (logits.argmax(dim=-1) == torch.argmax(y, dim=1)).float().sum()\n        count += len(y)\n\n      test_loss = torch.mean(torch.tensor(test_loss))\n      test_acc = correct / count\n\n      # Log test metrics\n      # wandb.log({\"test/loss\": test_loss, \"test/acc\": test_acc})\n\n      print()\n      print(f\"--- TEST ---\")\n      print(\"loss: \", test_loss.item())\n      print(\"accuracy: \", test_acc.item())\n  \n  model.train()\n  return array_loss_train, array_loss_val","metadata":{"id":"SB0MTaoYgTG7","execution":{"iopub.status.busy":"2022-04-04T16:43:10.973823Z","iopub.execute_input":"2022-04-04T16:43:10.974117Z","iopub.status.idle":"2022-04-04T16:43:11.001109Z","shell.execute_reply.started":"2022-04-04T16:43:10.974083Z","shell.execute_reply":"2022-04-04T16:43:11.000462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model5 = nn.Sequential(\n                      dataAugumentation,\n                      nn.Conv2d(3, 16, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(16),\n                      nn.Conv2d(16, 16, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(16),\n                      nn.MaxPool2d(2, 2),\n                      \n\n                      nn.Conv2d(16, 32, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(32),\n                      nn.Conv2d(32, 32, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(32),\n                      nn.MaxPool2d(2, 2),\n\n                      nn.Conv2d(32, 64, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(64),\n                      nn.Conv2d(64, 64, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(64),\n                      nn.MaxPool2d(2, 2),\n\n                      nn.Conv2d(64, 128, 3, 1, padding='same'),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.BatchNorm2d(128),\n                      nn.Conv2d(128, 128, 3, 1, padding='same'),\n                      nn.Dropout(hyp[\"dropout_rate\"]),\n                      nn.ReLU(),\n                      nn.BatchNorm2d(128),\n                      nn.MaxPool2d(2, 2),\n\n                      nn.Flatten(start_dim=-3),\n                      nn.Linear(512, 128),\n                      nn.ReLU(),\n                      nn.Dropout(hyp[\"dropout_rate\"] + 0.1),\n                      nn.Linear(128, 10),\n\n                ).cuda()","metadata":{"id":"dx1qCJQT0GjR","execution":{"iopub.status.busy":"2022-04-04T16:43:14.257967Z","iopub.execute_input":"2022-04-04T16:43:14.258829Z","iopub.status.idle":"2022-04-04T16:43:16.622197Z","shell.execute_reply.started":"2022-04-04T16:43:14.258770Z","shell.execute_reply":"2022-04-04T16:43:16.621438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"summary(cnn_model5, (3, 32, 32))","metadata":{"id":"M55uXc4zRZmI","outputId":"b9723476-8758-4359-ed4b-ae434638d545","execution":{"iopub.status.busy":"2022-04-04T16:43:19.586689Z","iopub.execute_input":"2022-04-04T16:43:19.586977Z","iopub.status.idle":"2022-04-04T16:43:19.617940Z","shell.execute_reply.started":"2022-04-04T16:43:19.586945Z","shell.execute_reply":"2022-04-04T16:43:19.617228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"init_weights = copy.deepcopy(cnn_model5.state_dict())","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:43:22.769477Z","iopub.execute_input":"2022-04-04T16:43:22.769723Z","iopub.status.idle":"2022-04-04T16:43:22.786109Z","shell.execute_reply.started":"2022-04-04T16:43:22.769695Z","shell.execute_reply":"2022-04-04T16:43:22.785452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_loss, val_los = train(cnn_model5, train_dataloader, val_dataloader,num_epochs=100, lr = hyp[\"lerning_rate\"])","metadata":{"id":"n7BSp-2yR1eS","outputId":"4c3307ac-1720-4fb0-ac92-883a0ad49a10","execution":{"iopub.status.busy":"2022-04-04T16:43:24.927247Z","iopub.execute_input":"2022-04-04T16:43:24.927775Z","iopub.status.idle":"2022-04-04T16:47:41.271603Z","shell.execute_reply.started":"2022-04-04T16:43:24.927739Z","shell.execute_reply":"2022-04-04T16:47:41.270888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"figure(figsize=(15,11), dpi=80)\nplt.plot(range(1,101), tr_loss,label = \"Train loss\")\nplt.plot(range(1,101), val_los, label = \"Validation loss\")\nplt.title(hyp)\nplt.legend()","metadata":{"id":"Ruoa-vodoXjL","outputId":"89866f6d-001e-4249-8b19-abc6684c9368","execution":{"iopub.status.busy":"2022-04-04T16:48:27.095391Z","iopub.execute_input":"2022-04-04T16:48:27.095779Z","iopub.status.idle":"2022-04-04T16:48:27.361895Z","shell.execute_reply.started":"2022-04-04T16:48:27.095737Z","shell.execute_reply":"2022-04-04T16:48:27.361225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs_num = np.argmin(val_los)\nprint(\"Epoch hyperparameter \" + str(epochs_num))","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:32.354797Z","iopub.execute_input":"2022-04-04T16:48:32.355071Z","iopub.status.idle":"2022-04-04T16:48:32.360075Z","shell.execute_reply.started":"2022-04-04T16:48:32.355041Z","shell.execute_reply":"2022-04-04T16:48:32.359253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best val loss\")\nval_los[epochs_num]","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:35.227910Z","iopub.execute_input":"2022-04-04T16:48:35.228440Z","iopub.status.idle":"2022-04-04T16:48:35.235059Z","shell.execute_reply.started":"2022-04-04T16:48:35.228399Z","shell.execute_reply":"2022-04-04T16:48:35.234254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prediction(model,test_dataloader):\n  with torch.no_grad():\n    model.eval()\n    prediction = np.empty([0], dtype=np.int8)\n\n\n    for test_batch in tqdm(test_dataloader, desc=\"Testing\"):\n      x, _ = test_batch\n      x = x.float().cuda()\n      logits = model(x)\n      tmp_arr = logits.argmax(dim=-1)\n      prediction = np.append(prediction,tmp_arr.cpu().numpy())\n  return prediction","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:40.104768Z","iopub.execute_input":"2022-04-04T16:48:40.105318Z","iopub.status.idle":"2022-04-04T16:48:40.111607Z","shell.execute_reply.started":"2022-04-04T16:48:40.105277Z","shell.execute_reply":"2022-04-04T16:48:40.110701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cifar10_dataloader_noShuffle = DataLoader(val_datasets, batch_size=hyp[\"batch_size\"], shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:43.671029Z","iopub.execute_input":"2022-04-04T16:48:43.671859Z","iopub.status.idle":"2022-04-04T16:48:43.676361Z","shell.execute_reply.started":"2022-04-04T16:48:43.671798Z","shell.execute_reply":"2022-04-04T16:48:43.675572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p = prediction(cnn_model5, cifar10_dataloader_noShuffle)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:46.118427Z","iopub.execute_input":"2022-04-04T16:48:46.118899Z","iopub.status.idle":"2022-04-04T16:48:46.349513Z","shell.execute_reply.started":"2022-04-04T16:48:46.118862Z","shell.execute_reply":"2022-04-04T16:48:46.348370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p_encoded = cifar10_dataset.encodeLabel(p)\np2 = np.array([int(np.argmax(i[1])) for i in val_datasets])\np2_encoded = cifar10_dataset.encodeLabel(p2)\ncm = confusion_matrix(p_encoded, p2_encoded)\ncm","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:48:48.051700Z","iopub.execute_input":"2022-04-04T16:48:48.052617Z","iopub.status.idle":"2022-04-04T16:48:48.529838Z","shell.execute_reply.started":"2022-04-04T16:48:48.052567Z","shell.execute_reply":"2022-04-04T16:48:48.529044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\nfigure(figsize=(15,11), dpi=80)\n\nax = sns.heatmap(cm, annot=True, cmap='Blues')\n\nax.set_title('simple CNN\\n\\n');\nax.set_xlabel('\\nPredicted Values')\nax.set_ylabel('Actual Values ');\n\n## Ticket labels - List must be in alphabetical order\nax.xaxis.set_ticklabels(cifar10_dataset.encodeLabel(list(range(10))))\nax.yaxis.set_ticklabels(cifar10_dataset.encodeLabel(list(range(10))))\n\n## Display the visualization of the Confusion Matrix.\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:49:14.511799Z","iopub.execute_input":"2022-04-04T16:49:14.512389Z","iopub.status.idle":"2022-04-04T16:49:15.120223Z","shell.execute_reply.started":"2022-04-04T16:49:14.512348Z","shell.execute_reply":"2022-04-04T16:49:15.119517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from itertools import chain\nfrom sklearn.utils import _safe_indexing\n\nindexes = np.arange(len(cifar10_dataset))\ncifar10_Whole = list(\n        chain.from_iterable(\n            (_safe_indexing(a, indexes)) for a in [cifar10_dataset]\n        )\n      )","metadata":{"id":"J9cHbj-J77hn","execution":{"iopub.status.busy":"2022-04-04T16:49:38.867375Z","iopub.execute_input":"2022-04-04T16:49:38.867634Z","iopub.status.idle":"2022-04-04T16:49:55.238971Z","shell.execute_reply.started":"2022-04-04T16:49:38.867606Z","shell.execute_reply":"2022-04-04T16:49:55.238241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cifar10_dataloader_Whole = DataLoader(cifar10_Whole, batch_size=hyp[\"batch_size\"], shuffle=hyp[\"shuffle\"])","metadata":{"id":"mBXDKIjV_164","execution":{"iopub.status.busy":"2022-04-04T16:49:59.590761Z","iopub.execute_input":"2022-04-04T16:49:59.591022Z","iopub.status.idle":"2022-04-04T16:49:59.597583Z","shell.execute_reply.started":"2022-04-04T16:49:59.590993Z","shell.execute_reply":"2022-04-04T16:49:59.596823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnn_model5.load_state_dict(init_weights)","metadata":{"execution":{"iopub.status.busy":"2022-04-04T16:50:01.579916Z","iopub.execute_input":"2022-04-04T16:50:01.580161Z","iopub.status.idle":"2022-04-04T16:50:01.589551Z","shell.execute_reply.started":"2022-04-04T16:50:01.580134Z","shell.execute_reply":"2022-04-04T16:50:01.588733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_loss, _ = train(cnn_model5, cifar10_dataloader_Whole,num_epochs=100,lr = hyp[\"lerning_rate\"])","metadata":{"id":"urgfJ-3m831P","outputId":"45d68e96-bccc-4ab9-8db8-9fd6d77595ac","execution":{"iopub.status.busy":"2022-04-04T16:50:03.588319Z","iopub.execute_input":"2022-04-04T16:50:03.588991Z","iopub.status.idle":"2022-04-04T16:54:55.685164Z","shell.execute_reply.started":"2022-04-04T16:50:03.588951Z","shell.execute_reply":"2022-04-04T16:54:55.684372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cifar10_dataset_test = CustomImageDataset(\"./test/\")","metadata":{"id":"0gA95w3FM5jN","execution":{"iopub.status.busy":"2022-04-04T17:10:32.552880Z","iopub.execute_input":"2022-04-04T17:10:32.553640Z","iopub.status.idle":"2022-04-04T17:10:32.561002Z","shell.execute_reply.started":"2022-04-04T17:10:32.553591Z","shell.execute_reply":"2022-04-04T17:10:32.560278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(cifar10_dataset_test, batch_size=hyp[\"batch_size\"], shuffle=False)\n","metadata":{"id":"NjJxnQ7lNmIO","execution":{"iopub.status.busy":"2022-04-04T17:10:34.071889Z","iopub.execute_input":"2022-04-04T17:10:34.072180Z","iopub.status.idle":"2022-04-04T17:10:34.077231Z","shell.execute_reply.started":"2022-04-04T17:10:34.072147Z","shell.execute_reply":"2022-04-04T17:10:34.076019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" p_test = prediction(cnn_model5, test_dataloader)","metadata":{"id":"qElyhMqhnrIE","execution":{"iopub.status.busy":"2022-04-04T17:10:35.580808Z","iopub.execute_input":"2022-04-04T17:10:35.581086Z","iopub.status.idle":"2022-04-04T17:11:44.107313Z","shell.execute_reply.started":"2022-04-04T17:10:35.581055Z","shell.execute_reply":"2022-04-04T17:11:44.106485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preview(datasets,encoder, labels = None,index = None):\n  if index is not None:\n    figure(figsize=(8, 6), dpi=80)\n    img, label = datasets[index]\n    if labels is not None:\n      label = labels[index]\n    plt.imshow(img.int().T)\n    plt.show()\n    print(encoder([label]))\n    print()\n    return\n\n  visited = []\n\n  for i, data in enumerate(datasets): \n    img, label = data\n    if labels is not None:\n      label = labels[i]\n    if label in visited:\n      continue\n    figure(figsize=(8, 6), dpi=80)\n    plt.imshow(img.int().T)\n    plt.show()\n    print(encoder([label]))\n    print()\n    visited.append(label)\n    if len(visited) >= 10:\n      break\n    \n    ","metadata":{"id":"XKIq6bLh-JSw","execution":{"iopub.status.busy":"2022-04-04T17:11:51.867862Z","iopub.execute_input":"2022-04-04T17:11:51.868114Z","iopub.status.idle":"2022-04-04T17:11:51.876730Z","shell.execute_reply.started":"2022-04-04T17:11:51.868086Z","shell.execute_reply":"2022-04-04T17:11:51.875893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preview(cifar10_dataset_test,lambda x: cifar10_dataset.encodeLabel(x),p_test)","metadata":{"id":"HLShwgg5o_0P","execution":{"iopub.status.busy":"2022-04-04T17:11:53.964480Z","iopub.execute_input":"2022-04-04T17:11:53.965028Z","iopub.status.idle":"2022-04-04T17:11:55.682727Z","shell.execute_reply.started":"2022-04-04T17:11:53.964989Z","shell.execute_reply":"2022-04-04T17:11:55.682074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame(data=cifar10_dataset.encodeLabel(p_test))\ndf.columns = [\"label\"]\ndf.index += 1 ","metadata":{"id":"JsNZJrZgEeXJ","execution":{"iopub.status.busy":"2022-04-04T17:12:03.574280Z","iopub.execute_input":"2022-04-04T17:12:03.574542Z","iopub.status.idle":"2022-04-04T17:12:03.590446Z","shell.execute_reply.started":"2022-04-04T17:12:03.574513Z","shell.execute_reply":"2022-04-04T17:12:03.589772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv('submission.csv', index=True, index_label=\"id\")","metadata":{"id":"sSRejCbBFiRT","execution":{"iopub.status.busy":"2022-04-04T17:12:06.048539Z","iopub.execute_input":"2022-04-04T17:12:06.048787Z","iopub.status.idle":"2022-04-04T17:12:06.611809Z","shell.execute_reply.started":"2022-04-04T17:12:06.048757Z","shell.execute_reply":"2022-04-04T17:12:06.611097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!kaggle competitions submit -c cifar-10 -f ../kaggle/working/submission.csv -m \"Deep Learning, PW, First try\"\n\n","metadata":{"id":"51tg9anDH0Y_","execution":{"iopub.status.busy":"2022-04-02T14:45:11.340428Z","iopub.execute_input":"2022-04-02T14:45:11.341018Z","iopub.status.idle":"2022-04-02T14:45:12.498628Z","shell.execute_reply.started":"2022-04-02T14:45:11.340978Z","shell.execute_reply":"2022-04-02T14:45:12.497626Z"},"trusted":true},"execution_count":null,"outputs":[]}]}