{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Semi Supervised GAN for image classification\n\nPytorch implementation\n\n## The Data\n\nUses [DALI by NVIDIA](https://github.com/NVIDIA/DALI) to speedup the pre-processing (namely, resizing) of the images","metadata":{}},{"cell_type":"code","source":"# !pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --extra-index-url https://developer.download.nvidia.com/compute/redist/cuda/11.0 nvidia-dali==0.22.0\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:04:45.307796Z","iopub.execute_input":"2022-05-05T18:04:45.308046Z","iopub.status.idle":"2022-05-05T18:05:10.719479Z","shell.execute_reply.started":"2022-05-05T18:04:45.308018Z","shell.execute_reply":"2022-05-05T18:05:10.718635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import print_function, division\n\nimport random\nfrom random import shuffle\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport copy\n\nimport os\nfrom os import listdir\nfrom os.path import join\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nimport torch.nn.functional as F\n\nimport torchvision\nimport torchvision.utils as vutils\nfrom torchvision import utils, models\n\nfrom nvidia.dali.pipeline import Pipeline\nimport nvidia.dali.ops as ops\nimport nvidia.dali.types as types\nfrom nvidia.dali.plugin.pytorch import DALIGenericIterator, DALIClassificationIterator\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\n# len(os.listdir('./data/train'))\n\nimport pandas as pd\n\nfrom PIL import Image\nfrom zipfile import ZipFile\n\nimport shutil","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.705908Z","iopub.execute_input":"2022-05-05T18:05:41.706346Z","iopub.status.idle":"2022-05-05T18:05:41.716315Z","shell.execute_reply.started":"2022-05-05T18:05:41.706299Z","shell.execute_reply":"2022-05-05T18:05:41.715603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_path_train = '/kaggle/input/dogs-vs-cats/train.zip'\nwith ZipFile(zip_path_train) as myzip:\n    files_in_zip_train = myzip.namelist()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:18.004175Z","iopub.execute_input":"2022-05-05T18:05:18.004783Z","iopub.status.idle":"2022-05-05T18:05:18.347140Z","shell.execute_reply.started":"2022-05-05T18:05:18.004696Z","shell.execute_reply":"2022-05-05T18:05:18.346397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zip_path_train = '/kaggle/input/dogs-vs-cats/test1.zip'\nwith ZipFile(zip_path_train) as myzip:\n    files_in_zip_test = myzip.namelist()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:18.348438Z","iopub.execute_input":"2022-05-05T18:05:18.348687Z","iopub.status.idle":"2022-05-05T18:05:18.466143Z","shell.execute_reply.started":"2022-05-05T18:05:18.348657Z","shell.execute_reply":"2022-05-05T18:05:18.465490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# extracting all files\nbase_dir = \"/kaggle/input/dogs-vs-cats/\"\ntrain_dir = os.path.join(base_dir, \"train.zip\")\ntest_dir = os.path.join(base_dir, \"test1.zip\")\n\nimport zipfile\nwith zipfile.ZipFile(train_dir,\"r\") as z:\n    z.extractall()\n\nwith zipfile.ZipFile(test_dir,\"r\") as z:\n    z.extractall()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:25.198960Z","iopub.execute_input":"2022-05-05T18:05:25.199204Z","iopub.status.idle":"2022-05-05T18:05:41.697325Z","shell.execute_reply.started":"2022-05-05T18:05:25.199177Z","shell.execute_reply":"2022-05-05T18:05:41.696569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('./data/labelled')\nos.makedirs('./data/train')\nos.makedirs('./data/test')\nos.makedirs('./data/validate')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.698916Z","iopub.execute_input":"2022-05-05T18:05:41.699166Z","iopub.status.idle":"2022-05-05T18:05:41.704512Z","shell.execute_reply.started":"2022-05-05T18:05:41.699133Z","shell.execute_reply":"2022-05-05T18:05:41.703787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#labelled dataset -> 100 images\n\n#cats\nfor filename in files_in_zip_train[1:51]:\n       shutil.move(os.path.join(filename),'./data/labelled')\n\n#dogs\nfor filename in files_in_zip_train[-50:]:\n       shutil.move(os.path.join(filename),'./data/labelled')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.718489Z","iopub.execute_input":"2022-05-05T18:05:41.718845Z","iopub.status.idle":"2022-05-05T18:05:41.726780Z","shell.execute_reply.started":"2022-05-05T18:05:41.718810Z","shell.execute_reply":"2022-05-05T18:05:41.726102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#validate set 1000 from labelled   \nfor filename in files_in_zip_train[51:551]:\n       shutil.move(os.path.join(filename),'./data/validate')\n        \nfor filename in files_in_zip_train[-551:-51]:\n       shutil.move(os.path.join(filename),'./data/validate')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.727902Z","iopub.execute_input":"2022-05-05T18:05:41.728281Z","iopub.status.idle":"2022-05-05T18:05:41.771214Z","shell.execute_reply.started":"2022-05-05T18:05:41.728245Z","shell.execute_reply":"2022-05-05T18:05:41.770496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test set 1000 from labelled    \nfor filename in files_in_zip_train[551:1051]:\n       shutil.move(os.path.join(filename),'./data/test')\n        \nfor filename in files_in_zip_train[-1051:-551]:\n       shutil.move(os.path.join(filename),'./data/test')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.772495Z","iopub.execute_input":"2022-05-05T18:05:41.772847Z","iopub.status.idle":"2022-05-05T18:05:41.815361Z","shell.execute_reply.started":"2022-05-05T18:05:41.772810Z","shell.execute_reply":"2022-05-05T18:05:41.814622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train -> remaining labelled and unlabelled data\nfor filename in files_in_zip_train[1051:-1051]:\n       shutil.move(os.path.join(filename),'./data/train')\n        \nfor filename in files_in_zip_test[1:]:\n       shutil.move(os.path.join(filename),'./data/train')","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:41.816918Z","iopub.execute_input":"2022-05-05T18:05:41.817188Z","iopub.status.idle":"2022-05-05T18:05:43.050648Z","shell.execute_reply.started":"2022-05-05T18:05:41.817154Z","shell.execute_reply":"2022-05-05T18:05:43.049840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('./data/train'))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:43.051908Z","iopub.execute_input":"2022-05-05T18:05:43.052178Z","iopub.status.idle":"2022-05-05T18:05:43.081716Z","shell.execute_reply.started":"2022-05-05T18:05:43.052143Z","shell.execute_reply":"2022-05-05T18:05:43.080835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('./data/labelled'))","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:43.082952Z","iopub.execute_input":"2022-05-05T18:05:43.083281Z","iopub.status.idle":"2022-05-05T18:05:43.089855Z","shell.execute_reply.started":"2022-05-05T18:05:43.083242Z","shell.execute_reply":"2022-05-05T18:05:43.089043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('./data/test'))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:43.093078Z","iopub.execute_input":"2022-05-05T18:05:43.093429Z","iopub.status.idle":"2022-05-05T18:05:43.102101Z","shell.execute_reply.started":"2022-05-05T18:05:43.093390Z","shell.execute_reply":"2022-05-05T18:05:43.101227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(os.listdir('./data/validate'))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:43.104085Z","iopub.execute_input":"2022-05-05T18:05:43.104699Z","iopub.status.idle":"2022-05-05T18:05:43.111897Z","shell.execute_reply.started":"2022-05-05T18:05:43.104658Z","shell.execute_reply":"2022-05-05T18:05:43.110872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.random.seed(42)\nrandom.seed(10)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\ntorch.manual_seed(999)\n\ndevice = torch.device(\"cuda\")\n\nlabeled_train_dir = './data/labelled'\nlabeled_batch = 100 # labeled_batch = total number of labeled training samples since it is small enough\nunlabeled_train_dir = './data/train'\nval_dir = './data/validate'\ntest_dir = './data/test'\n\ndef is_jpeg(filename):\n    return any(filename.endswith(extension) for extension in [\".jpg\", \".jpeg\"])\n\nclass ExternalInputIterator(object):\n\n    def __init__(self, imageset_dir, labeled, batch_size, random_shuffle=False):\n        self.images_dir = imageset_dir\n        self.batch_size = batch_size\n        self.labeled = labeled # True is images are labeled, False otherwise\n\n        self.image_files = np.array([join(imageset_dir, file_name) for file_name in sorted(listdir(imageset_dir)) if is_jpeg(file_name)])\n        \n        # In my dataset, filenames are of the form \"cat_***.jpg\" or \"*cat_***.jpg\" (same for dogs)\n        # The map below infers the image's label from its the second character of its filename\n        self.class_map = {'a': 0, 'o': 1, 'c': 0, 'd': 1}\n        self.offset = len(self.images_dir) + 2\n        \n        if random_shuffle:\n            ind = np.array(range(len(self.image_files)))\n            shuffle(self.image_files)\n\n    def __iter__(self):\n        self.i = 0\n        self.n = len(self.image_files)\n        return self\n\n\n    # Return a batch of (input, target) pairs\n    def __next__(self):\n        \n        images = []\n        labels = []\n\n        for _ in range(self.batch_size):\n            image_file = self.image_files[self.i]\n            if self.labeled:\n                label = self.class_map[self.image_files[self.i][self.offset]]\n                labels.append(np.array([label], dtype = np.uint8))\n            image = open(image_file, 'rb')\n            images.append(np.frombuffer(image.read(), dtype = np.uint8))\n            image.close()\n\n            self.i = (self.i + 1) % self.n\n        if self.labeled:\n            return (images, labels)\n        return images\n\n    next = __next__\n\nclass PetPipeline(Pipeline):\n    '''\n    Constructor arguments:  \n    - imageset_dir: directory containing the dataset\n    - labeled = True\n    - image_size = 128: length of the square that the images will be resized to\n    - random_shuffle = False\n    - batch_size = 64\n    - num_threads = 1\n    - device_id = 0\n    '''\n\n    def __init__(self, imageset_dir, labeled=True, image_size=128, random_shuffle=False, batch_size=64, num_threads=1, device_id=0):\n        super(PetPipeline, self).__init__(batch_size, num_threads, device_id, seed=12)  \n        eii = ExternalInputIterator(imageset_dir, labeled, batch_size, random_shuffle)\n        self.iterator = iter(eii)\n        self.num_inputs = len(eii.image_files)\n        self.labeled = labeled\n\n        # The source for the inputs\n        self.input = ops.ExternalSource()\n        self.input_label = ops.ExternalSource()\n\n        self.decode = ops.ImageDecoder(device = 'mixed', output_type = types.RGB)\n\n        # The rest of pre-processing is done on the GPU\n        self.res = ops.Resize(device=\"gpu\", resize_x=image_size, resize_y=image_size)\n        self.norm = ops.CropMirrorNormalize(device='gpu', mean=0, std=255.0)\n\n\n    # epoch_size = number of (profile, frontal) image pairs in the dataset\n    def epoch_size(self, name = None):\n        return self.num_inputs\n\n\n    # Define the flow of the data loading and pre-processing\n    def define_graph(self):   \n        \n        if self.labeled:\n            self.labels = self.input_label()\n        self.jpegs = self.input()\n        images = self.decode(self.jpegs)\n        images = self.res(images)\n        output = self.norm(images)\n        if self.labeled:\n            self.labels = self.input_label()\n            return (output, self.labels)\n        else:\n            return output\n    \n    def iter_setup(self):\n        if self.labeled:\n            (images, labels) = self.iterator.next()\n            self.feed_input(self.jpegs, images)\n            self.feed_input(self.labels, labels)\n        else:\n            images = self.iterator.next()\n            self.feed_input(self.jpegs, images)\n    \n# Test labeled and unlabeled dataloaders:\n    \ndata_pipe = PetPipeline(labeled_train_dir, True, image_size=224, random_shuffle=True, batch_size=16)\ndata_pipe.build()\n# Use DALIClassificationIterator for the labeled Pipeline\ndata_loader = DALIClassificationIterator(data_pipe, data_pipe.epoch_size())\n\ntest_batch = next(iter(data_loader))\nimages_batch = test_batch[0]['data'].cpu()\nlabels_batch = test_batch[0]['label']\nfig = plt.figure(figsize=(20,10))\ngrid = utils.make_grid(images_batch)\nplt.imshow(grid.numpy().transpose((1, 2, 0)))\nplt.show()\nprint('Labels: ', labels_batch.T)\n\ndata_pipe = PetPipeline(labeled_train_dir, False, image_size=224, random_shuffle=True, batch_size=16)\ndata_pipe.build()\n# Use DALIGenericIterator for the unlabeled Pipeline\ndata_loader = DALIGenericIterator(data_pipe, ['data'], data_pipe.epoch_size())\n\ntest_batch = next(iter(data_loader))\nimages_batch = test_batch[0]['data'].cpu()\nfig = plt.figure(figsize=(20,10))\ngrid = utils.make_grid(images_batch)\nplt.imshow(grid.numpy().transpose((1, 2, 0)))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:49.385513Z","iopub.execute_input":"2022-05-05T18:05:49.385736Z","iopub.status.idle":"2022-05-05T18:05:54.011190Z","shell.execute_reply.started":"2022-05-05T18:05:49.385711Z","shell.execute_reply":"2022-05-05T18:05:54.010484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Supervised ResNet18 trained on 100 training samples","metadata":{}},{"cell_type":"code","source":"# test_pipe = PetPipeline(labeled_train_dir, True, image_size=224, random_shuffle=True, batch_size=512)\ntest_pipe = PetPipeline(test_dir, True, image_size=224, random_shuffle=True, batch_size=100)\n\ntest_pipe.build()\n\nm_test = test_pipe.epoch_size()\n\ntest_loader = DALIClassificationIterator(test_pipe, m_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:54.657852Z","iopub.execute_input":"2022-05-05T18:05:54.658358Z","iopub.status.idle":"2022-05-05T18:05:54.749507Z","shell.execute_reply.started":"2022-05-05T18:05:54.658322Z","shell.execute_reply":"2022-05-05T18:05:54.748680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# final_test_pipe = PetPipeline(test_dir, True, image_size=224, random_shuffle=True, batch_size=100)\n# final_test_pipe.build()\n# final_test_loader = DALIClassificationIterator(final_test_pipe, final_test_pipe.epoch_size())\n\nlabeled_pipe = PetPipeline(labeled_train_dir, True, image_size=224, random_shuffle=False, batch_size=labeled_batch)\nlabeled_pipe.build()\nm_l = labeled_pipe.epoch_size()\nlabeled_loader = DALIClassificationIterator(labeled_pipe, m_l)\n\nlabeled_batch = next(iter(labeled_loader))\nlabeled_data = labeled_batch[0]['data'].type('torch.FloatTensor').to(device)\nlabels = labeled_batch[0]['label'].type(dtype=torch.long).squeeze().to(device)\n\ndef test(model, device, test_loader, m_test, display = False):\n    model.eval()\n    test_loss = 0\n    correct = 0\n    with torch.no_grad():\n        for batch in test_loader:\n            data = batch[0]['data'].type('torch.FloatTensor').to(device)\n            target = batch[0]['label'].type(dtype=torch.long).squeeze().to(device)\n            output = model(data)\n            test_loss += criterion(output, target).item() \n            pred = output.argmax(dim=1, keepdim=True)\n            correct += pred.eq(target.view_as(pred)).sum().item()\n\n    test_loss /= m_test\n\n    if display == True:\n        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n        test_loss, correct, m_test,\n        100. * correct / m_test))\n        \n    test_loader.reset()    \n    return test_loss, 100. * correct / m_test\n\n\nepochs = 100\nlr = 0.1\ngamma = 0.1\n\n# Use a (not pre-trained) ResNet18 with d_out = 2\nclassifier = models.resnet18()\nnum_ftrs = classifier.fc.in_features\nclassifier.fc = nn.Linear(num_ftrs, 2)\nclassifier = classifier.to(device)\n\noptimizerC = optim.SGD(classifier.parameters(), lr=0.001, momentum=0.9, dampening=0, weight_decay=0.0001)\n\ncriterion = nn.CrossEntropyLoss()\n\nepochs = 200\n\ntest_losses = []\ntest_accuracies = []\n\nbest_model_wts = copy.deepcopy(classifier.state_dict())\nbest_acc = 0.0\n\nfor epoch in range(1, epochs + 1):\n    \n    classifier.train()\n    \n    optimizerC.zero_grad()\n    output = classifier(labeled_data)\n    loss = criterion(output, labels)\n    loss.backward()\n    \n    if epoch % 50 == 0:\n        print('Train Epoch: {} \\tLoss: {:.6f}'.format(epoch, loss.item()))\n    \n        test_loss, test_accuracy = test(classifier, device, test_loader, m_test, True)\n        test_losses.append(test_loss)\n        test_accuracies.append(test_accuracy)\n    else:\n        test_loss, test_accuracy = test(classifier, device, test_loader, m_test, False)\n        test_losses.append(test_loss)\n        test_accuracies.append(test_accuracy)\n        \n    if test_accuracy > best_acc:\n        best_acc = test_accuracy\n        best_classifier_wts = copy.deepcopy(classifier.state_dict())\n        \n    optimizerC.step()\n        \nplt.plot(range(len(test_losses)), np.array(test_losses))\nplt.show()\n\nprint('Best VAL test accuracy: ', np.max(np.array(test_accuracies)),\n      '% after ', np.argmax(np.array(test_accuracies)), ' training epochs')\n\n# load best model weights\nclassifier.load_state_dict(best_classifier_wts)\n\nprint(\"\\n Best model on the test set: \")\n# test(classifier, device, final_test_loader, final_test_pipe.epoch_size(), True)\ntest(classifier, device, test_loader, test_pipe.epoch_size(), True)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:05:57.055923Z","iopub.execute_input":"2022-05-05T18:05:57.056185Z","iopub.status.idle":"2022-05-05T18:12:10.176261Z","shell.execute_reply.started":"2022-05-05T18:05:57.056156Z","shell.execute_reply":"2022-05-05T18:12:10.175529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Semi supervised GAN\n\n* Generator + ResNet18 classifier used as a Discriminator\n* 100 labeled training samples\n* Around 30K unlabeled images","metadata":{}},{"cell_type":"code","source":"batch_size = 64\n\nepochs = 14\n\ntrain_pipe = PetPipeline(unlabeled_train_dir, False, image_size=224, random_shuffle=True, batch_size=batch_size)\ntrain_pipe.build()\nm_train = train_pipe.epoch_size()\ntrain_loader = DALIGenericIterator(train_pipe, ['data'], m_train)\n\ndef log_sum_exp(x, axis = 1):\n    m = torch.max(x, dim = 1)[0]\n    return m + torch.log(torch.sum(torch.exp(x - m.unsqueeze(1)), dim = axis))\n\n# custom weights initialization called on the Generator\ndef weights_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        nn.init.normal_(m.weight.data, 0.0, 0.02)\n    elif classname.find('BatchNorm') != -1:\n        nn.init.normal_(m.weight.data, 1.0, 0.02)\n        nn.init.constant_(m.bias.data, 0)\n        \n\n# Number of channels in the training images. For color images this is 3\nnc = 3\n# Size of z latent vector \nnz = 100\n# Size of feature maps in generator\nngf = 64\n\nclass Generator(nn.Module):\n    def __init__(self):\n        super(Generator, self).__init__()\n        self.main = nn.Sequential(\n            # input is Z, going into a convolution\n            nn.ConvTranspose2d( nz, ngf * 8, 7, 1, 0, bias=False),\n            nn.BatchNorm2d(ngf * 8),\n            nn.ReLU(True),\n            # state size. (ngf*8) x 7 x 7\n            nn.ConvTranspose2d(ngf * 8, ngf * 4, 4, 2, 1, bias=False),\n            nn.BatchNorm2d(ngf * 4),\n            nn.ReLU(True),\n            # state size. (ngf*4) x 14 x 14\n            nn.ConvTranspose2d( ngf * 4, ngf * 2, 4, 2, 1, bias=False),\n            nn.BatchNorm2d(ngf * 2),\n            nn.ReLU(True),\n            # state size. (ngf*2) x 28 x 28\n            nn.ConvTranspose2d( ngf * 2, ngf, 4, 4, 0, bias=False),\n            nn.BatchNorm2d(ngf),\n            nn.ReLU(True),\n            # state size. (ngf) x 112 x 112\n            nn.ConvTranspose2d( ngf, nc, 4, 2, 1, bias=False),\n            nn.Tanh()\n            # state size. (nc) x 224 x 224\n        )\n\n    def forward(self, input):\n        return self.main(input)\n    \n# A noise vector to be used for generating images at the end of each training epoch\nfixed_noise = torch.randn(64, nz, 1, 1, device=device)\n\nnetG = Generator().to(device)\nnetG.apply(weights_init)\n\nclassifier = models.resnet18()\nnum_ftrs = classifier.fc.in_features\nclassifier.fc = nn.Linear(num_ftrs, 2)\nclassifier = classifier.to(device)\n\noptimizerG = optim.Adam(netG.parameters(), lr=0.002, betas= (0.5, 0.999))\noptimizerC = optim.Adam(classifier.parameters(), lr=0.002, betas= (0.5, 0.999))#, dampening=0, weight_decay=0.0001)\n\ntest_losses = []\ntest_accuracies = []\n\nbest_model_wts = copy.deepcopy(classifier.state_dict())\nbest_acc = 0.0\n\nfor epoch in range(1, epochs + 1):\n\n    for batch_idx, batch in enumerate(train_loader):\n        \n        # TRAIN THE DISCRIMINATOR (THE CLASSIFIER)\n        classifier.train()\n        optimizerC.zero_grad()\n        \n        # 1. on Unlabelled data\n        data = batch[0]['data'].type('torch.FloatTensor').to(device)        \n        outputs = classifier(data)    \n        logz_unlabel = log_sum_exp(outputs)\n        lossUL = 0.5 * (-torch.mean(logz_unlabel) + torch.mean(F.softplus(logz_unlabel)))\n        lossUL.backward()  \n        \n        # 2. on the generated data\n\n        noise = torch.randn(batch_size, 100, 1, 1, device=device)\n        generated = (netG(noise)+1.0)/2.0\n        outputs = classifier(generated.detach()) # detach() because we are not training G here\n        logz_fake = log_sum_exp(outputs)\n        lossD = 0.5*torch.mean(F.softplus(logz_fake))\n        lossD.backward()\n        \n        # 3. on labeled data\n        output = classifier(labeled_data)\n        logz_label = log_sum_exp(output)\n        prob_label = torch.gather(output, 1, labels.unsqueeze(1))\n        labeled_loss = -torch.mean(prob_label) + torch.mean(logz_label)\n        labeled_loss.backward()    \n\n        optimizerC.step()\n        \n        # TRAIN THE DISCRIMINATOR (THE CLASSIFIER)\n        netG.train()\n        optimizerG.zero_grad()\n        \n        outputs = classifier(generated)\n        logz_unlabel = log_sum_exp(outputs)\n        lossG = 0.5 * (-torch.mean(logz_unlabel) + torch.mean(F.softplus(logz_unlabel)))\n        lossG.backward()\n        optimizerG.step()\n            \n    train_loader.reset()\n    \n    generated = (netG(fixed_noise)+1.0)/2.0\n    vutils.save_image(generated.cpu().detach(), ('generated_%d.jpg' % epoch), normalize=True)\n    \n    test_loss, test_accuracy = test(classifier, device, test_loader, m_test, True)\n    test_losses.append(test_loss)\n    test_accuracies.append(test_accuracy)\n        \n    if test_accuracy > best_acc:\n        best_acc = test_accuracy\n        best_classifier_wts = copy.deepcopy(classifier.state_dict())\n\nplt.plot(range(len(test_losses)), np.array(test_losses))\nplt.show()\n\nprint('Best VAL test accuracy: ', np.max(np.array(test_accuracies)),\n      '% after ', np.argmax(np.array(test_accuracies)), ' training epochs')\n\n# load best model weights\nclassifier.load_state_dict(best_classifier_wts)\n\nprint(\"\\nBest model on the test set: \")\n# test(classifier, device, final_test_loader, final_test_pipe.epoch_size(), True)\ntest(classifier, device, test_loader, test_pipe.epoch_size(), True)","metadata":{"execution":{"iopub.status.busy":"2022-05-05T18:12:50.077623Z","iopub.execute_input":"2022-05-05T18:12:50.077938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}