{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#! pip install madgrad\n#! pip install efficientnet_pytorch","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import copy\nfrom numpy import asarray\nimport os\nimport glob\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.nn.init as init\nimport torchvision.models as models\nfrom torchvision import transforms\nimport torchvision.transforms as transforms\nimport torch.utils.data as data\nimport torchvision\nimport random\nimport math\nfrom torch.autograd import Variable\n#import matplotlib.pyplot as plt\n#from modules import *\n#import torchvision.models as models_pytorch\n#import h5py\n#import torch.optim as optim\n#import augmentations\nfrom torch.nn.functional import kl_div, softmax, log_softmax\nimport torch.optim as optim\nimport torch.utils.data as data\nfrom os.path import exists, join, split\nfrom os import listdir\nfrom os.path import join\nfrom PIL import Image, ImageFilter , ImageDraw\nimport PIL\nimport random\n#import madgrad \nimport matplotlib.pyplot as plt\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_class = 10 #number of classes\ntotal_steps = 2**20\neval_step = 1024\nthreshhold = 0.95\nnum_epochs = math.ceil(total_steps/eval_step)\nseed=111 #seed for the algorithm\nbatch_size = 32\nlabeled_batch_size = 64\nu=7\nunlabeled_batch_size = labeled_batch_size*u\nnum_train =100 # number of training image by classe\n#Validation set size\nvalid_size = 200\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## First let us define a CNN","metadata":{}},{"cell_type":"code","source":"\n\nbn_momentum = 0.9\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True)\n\n\ndef conv_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        init.xavier_uniform_(m.weight, gain=np.sqrt(2))\n        init.constant_(m.bias, 0)\n    elif classname.find('BatchNorm') != -1:\n        init.constant_(m.weight, 1)\n        init.constant_(m.bias, 0)\n\n\nclass WideBasic(nn.Module):\n    def __init__(self, in_planes, planes, dropout_rate, stride=1):\n        super(WideBasic, self).__init__()\n        self.bn1 = nn.BatchNorm2d(in_planes, momentum=bn_momentum)\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, bias=True)\n        self.dropout = nn.Dropout(p=dropout_rate)\n        self.bn2 = nn.BatchNorm2d(planes, momentum=bn_momentum)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=True)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=True),\n            )\n\n    def forward(self, x):\n        out = self.dropout(self.conv1(F.relu(self.bn1(x))))\n        out = self.conv2(F.relu(self.bn2(out)))\n        out += self.shortcut(x)\n\n        return out\n\n\nclass WideResNet(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out = self.linear(out)\n        \n        return out\n\nclass WideResNet_NDA(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_NDA, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        logit = self.linear(out)\n\n        return out,logit\n\n\n\n\nclass WideResNet_conf(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_conf, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n        self.fconfid =  nn.Linear(nStages[3], 1)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out_final = self.linear(out)\n        out_conf = self.fconfid(out)\n\n        return out_final,out_conf\n\n\nclass WideResNet_drop(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_drop, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.dropout1 = nn.Dropout2d(0.25)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.dropout2 = nn.Dropout2d(0.25)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.dropout3 = nn.Dropout2d(0.25)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.dropout4 = nn.Dropout2d(0.25)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.dropout1(out)\n        out = self.layer2(out)\n        out = self.dropout2(out)\n        out = self.layer3(out)\n        out = self.dropout3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out = self.dropout4(out)\n        out = self.linear(out)\n        \n\n        return out","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install efficientnet_pytorch\nfrom efficientnet_pytorch import EfficientNet\ndef loadEffNet(pretrained=False, b=0):\n    model  = None\n    if(pretrained):\n        model = EfficientNet.from_pretrained('efficientnet-b{}'.format(b))\n        for param in model.parameters():\n            param.requires_grad = False\n    else:\n        model =  EfficientNet.from_name('efficientnet-b{}'.format(b))\n    model._fc = nn.Linear(1280,10, bias=True)\n    model = model.cuda()\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## now let us define a dataset","metadata":{}},{"cell_type":"code","source":"from PIL import Image, ImageEnhance\nimport random\n\ndef AddBlur(img, factor):\n    return img.filter(ImageFilter.GaussianBlur(factor))\n\ndef AddBrightness(img,factor):\n    factor=max(factor,0.4)\n    return ImageEnhance.Brightness(img).enhance(factor)\n\ndef AddColor(img,factor):\n    return ImageEnhance.Color(img).enhance(factor)\n\ndef AddContrast(img,factor):\n    return ImageEnhance.Contrast(img).enhance(factor)\n\ndef AddCutout(img,length):\n    w, h = img.size\n    x0 = np.random.uniform(0, w)\n    y0 = np.random.uniform(0, h)\n    x0 = int(max(0, x0 - length / 2.))\n    y0 = int(max(0, y0 - length / 2.))\n    x1 = int(min(w, x0 + length))\n    y1 = int(min(h, y0 + length))\n    xy = (x0, y0, x1, y1)\n    # gray\n    color = (127, 127, 127)\n    img = img.copy()\n    ImageDraw.Draw(img).rectangle(xy, color)\n    return img\n\n\ndef AddRotation(img,factor):\n    return img.rotate(factor)\n    \ndef AddTranslationX(img,factor):\n    if random.random() > 0.5:\n        factor = -factor\n    factor = factor * img.size[0]\n    return img.transform(img.size, PIL.Image.AFFINE, (1,0,factor,0,1,0))\n\ndef AddTranslationY(img,factor):\n    if random.random() > 0.5:\n        factor = -factor\n    factor = factor * img.size[0]\n    return img.transform(img.size, PIL.Image.AFFINE, (1,0,0,0,1,factor))\n\nclass AddRandAugment:\n    def __init__(self,n,m):\n        assert n >= 1\n        assert 1 <= m <= 10\n        self.n = n\n        self.m = m\n        self.augmentations = AugmentationsPossibilities()\n    \n    def __call__(self,img):\n        choices = random.choices(self.augmentations,k=self.n)\n        for (aug,factor) in choices:\n            frac = random.randint(1,self.m)/self.m\n            img = aug(img,frac*factor)\n        return img\n\ndef AugmentationsPossibilities():\n    res = [(AddBrightness,1.2),\n          (AddContrast,1.8),\n          (AddCutout,8),\n          (AddRotation,15),\n          (AddTranslationX,0.3),\n          (AddTranslationY,0.3),\n          (AddBlur,0.8)]\n    return res\n\nclass TransformAug:\n    def __init__(self,mean,std):\n        self.weak = transforms.Compose([\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomCrop(size=32,padding=int(32*0.125),\n                                  padding_mode='reflect'),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=mean, std=std)])   \n        self.strong = transforms.Compose([\n            transforms.RandomHorizontalFlip(),\n            transforms.RandomCrop(size=32,\n                                  padding=int(32*0.125),\n                                  padding_mode='reflect'),\n            AddRandAugment(n=2, m=10),\n            transforms.ToTensor(),\n            transforms.Normalize(mean=mean, std=std)])\n        \n    def __call__(self,img):\n        return self.weak(img),self.strong(img)\n            ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Dataset_sub_CIFAR(data.Dataset):\n\n    def __init__(self, data_feature, data_target,transform,phase='label'):\n        self.data_feature = data_feature\n        self.data_target = data_target\n        self.transform = transform\n        self.phase=phase\n\n\n\n    def __len__(self):\n        return len(self.data_feature)\n\n    def __getitem__(self, index):\n        # load image as ndarray type (Height * Width * Channels)\n        # be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]\n        # in this example, i don't use ToTensor() method of torchvision.transforms\n        # so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)\n        if self.phase=='label':\n            data_feature = self.transform(Image.fromarray(np.uint8(self.data_feature[index])))\n            data_target =  self.data_target[index]\n            return data_feature, data_target\n\n        else:\n            data_feature = self.data_feature[index].float()\n            return data_feature\n\n\ndata_mean = (0.4914, 0.4822, 0.4465)\ndata_std = (0.2471, 0.2435, 0.2616)\n\ntransform_train = transforms.Compose([\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomCrop(size=32,\n                             padding=int(32*0.125),\n                             padding_mode='reflect'),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=data_mean, std=data_std)]\n    )\ntransform_train_Aug = TransformAug(mean=data_mean, std=data_std)\ntransform_val = transforms.Compose([\n        transforms.ToTensor(),\n        transforms.Normalize(mean=data_mean, std=data_std)\n    ])\ntransform_test=TransformAug(mean=data_mean, std=data_std)\n    \n    #Dataset loading\nCIFAR10_train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, transform=None, download=True)\nCIFAR10_test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, transform=None, download=True)\nnp.random.seed(seed=seed)\npermuation=np.random.permutation(len(CIFAR10_train_dataset.targets))\n\nOriginal_train_data_x = (CIFAR10_train_dataset.data)\nOriginal_train_data_y = np.array(CIFAR10_train_dataset.targets)\nOriginal_train_data_x = Original_train_data_x[permuation]\nOriginal_train_data_y = Original_train_data_y[permuation]\n\nOriginal_test_data_x = CIFAR10_test_dataset.data\nOriginal_test_data_y = np.array(CIFAR10_test_dataset.targets)\n\n\n\n    #Selection of 250 labeled images for training and 2000 for validation\n    \nincr_class = torch.zeros(num_class)\ntrain_idx_dico = {} #labeled images index dictionnary\n\nfor i in range(num_class):\n    train_idx_dico[str(i)] = []\n\nvalid_idx = np.zeros(num_class * valid_size, dtype=np.int32) #validation images indexes (2000)\nincr_t = 0\nincr_v = 0\nincrtotal = 0\n\nfor idx in range(len(Original_train_data_y)):\n    class_y = Original_train_data_y[idx]\n    incrtotal += 1\n\n    train_idx_dico[str(class_y)].append(idx)\n    incr_class[class_y] += 1 #count the number of image per class\n    incr_t += 1\n\n\ntrain_idx = np.zeros(num_class * num_train, dtype=np.int32) #train labeled images indexes (250)\nlist_train_id = []\nlist_unalabel_id = []\nvalid_idx = []\nunlabel_idx_dico = {}\nfor i in range(num_class):\n    unlabel_idx_dico[str(i)] = []\nfor i in range(num_class):\n    list_train_id = list_train_id + train_idx_dico[str(i)][0:num_train]\n    valid_idx =valid_idx + train_idx_dico[str(i)][num_train:num_train+valid_size]\n    list_unalabel_id = list_unalabel_id + train_idx_dico[str(i)][num_train+valid_size::]\n    unlabel_idx_dico[str(i)] = train_idx_dico[str(i)][num_train::]\n\n    #Get labeled and unlabeled data\n\nx_train = Original_train_data_x[[int(i) for i in list_train_id]]\ny_train = Original_train_data_y[[int(i) for i in list_train_id]]\n\nx_unlabeled = Original_train_data_x[[int(i) for i in list_unalabel_id]]\ny_unlabeled = Original_train_data_y[[int(i) for i in list_unalabel_id]]\n\n    #Get validation set data\nx_valid = Original_train_data_x[[int(i) for i in valid_idx]]\ny_valid = Original_train_data_y[[int(i) for i in valid_idx]]\n\n    # Printing the size of the training, validation and test sets\nprint('Number of training examples: ' + str(x_train.shape[0]))\nprint('Number of unlabeled examples: ' + str(x_unlabeled.shape[0]))\nprint('Number of validation examples: ' + str(x_valid.shape[0]))\n\nclasses = ('plane', 'car', 'bird', 'cat',\n               'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n    #Dataloader creation\n\ntest_loader = torch.utils.data.DataLoader(\n        Dataset_sub_CIFAR(Original_test_data_x, Original_test_data_y, transform=transform_test),\n        batch_size = unlabeled_batch_size,\n        shuffle=False, num_workers=2)\n\ntrain_loader = torch.utils.data.DataLoader(\n        Dataset_sub_CIFAR(x_train, y_train, transform=transform_train_Aug),\n        batch_size=labeled_batch_size,shuffle=True, num_workers=2) #num_workers = 2 ou 1\n\nvalid_loader = torch.utils.data.DataLoader(\n        Dataset_sub_CIFAR(x_valid, y_valid, transform=transform_val),\n        batch_size=batch_size,\n        shuffle=False, num_workers=2)\n    \nfinal_loader = torch.utils.data.DataLoader(\n        Dataset_sub_CIFAR(Original_test_data_x, Original_test_data_y, transform=transform_val),\n        batch_size = 64,\n        shuffle=False, num_workers=2) ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def showDiffAug(img,m):\n    choices = AugmentationsPossibilities()\n    fig, axs = plt.subplots(len(choices),m, figsize = (20,20))\n    j=0\n    for (aug,factor) in choices:\n        for i in range(1,m+1):\n            frac = i/m\n            newimg = aug(img,frac*factor)\n            newimg = asarray(newimg)\n            axs[j][i-1].imshow(newimg)\n            axs[j][i-1].set_title(aug.__name__+\" \"+str(i))\n        j+=1\nimg =Image.fromarray(Original_train_data_x[1])\nshowDiffAug(img,10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now we build the CNN and the optimizer\n","metadata":{}},{"cell_type":"code","source":"","metadata":{"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from math import cos, pi\ndef learning_rate_scheduler(init, epoch , num_epochs):\n    return init*cos(7*pi*epoch/(16*num_epochs))\n\n    return init*math.pow(0.1, optim_factor)\ndef getlambda(epoch, trainacc):\n    return 1\n    \"\"\"if(trainacc<0.8):\n        return 0\n    if epoch<=50 : \n        return 0\n    if epoch<=100:\n        return 1\n    if epoch<=150:\n        return 2\n    if epoch<=200:\n        return 3\n    if epoch<=250:\n        return 4\n    return 5\"\"\"\n\ndef test(epoch,net,testloader):\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(testloader):\n            \n            inputs, targets = inputs.cuda(), targets.cuda()\n            inputs, targets = Variable(inputs), Variable(targets)\n            outputs = net(inputs)\n            loss = criterion(outputs, targets)\n\n            test_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += targets.size(0)\n            correct += predicted.eq(targets.data).cpu().sum()\n\n        \n    acc = 100.*correct/total\n    print(\"\\n| Validation Epoch #%d\\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" %(epoch, loss.item(), acc))\n    return acc\n\n        ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\npretrained = False\n#Networks creation\ninitialnet = WideResNet(28,2, dropout_rate=0.2, num_classes=num_class)\ninitialnet = initialnet.cuda()\ninitialnet_save = WideResNet(28, 2, dropout_rate=0.2, num_classes=num_class) # model where to save the results\ninitialnet_save = initialnet_save.cuda()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = copy.deepcopy(initialnet_save)\nnet_save = copy.deepcopy(initialnet_save)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_acc=0\nlog_interval = 15\nlabeled_iter = iter(train_loader)\nbesttrainacc=0\nunlabeled_iter = iter(test_loader)\nlr = 0.03\nfor epoch in range(num_epochs):  \n    net.train()\n    optimizer = optim.SGD(net.parameters(), lr=learning_rate_scheduler(lr, epoch, num_epochs), momentum=0.9, weight_decay=5e-4, nesterov=True)\n\n    print('\\n=> Training Epoch #%d, LR=%.4f' %(epoch, learning_rate_scheduler(lr, epoch,num_epochs)))\n    train_loss = 0\n    correct = 0\n    total = 0\n    for batch_idx in range(eval_step):\n        try : \n            (x_t_w,x_t_s), y = labeled_iter.next()\n        except:\n            labeled_iter = iter(train_loader)\n            (x_t_w,x_t_s), y = labeled_iter.next()\n        try : \n            (x_w, x_s), _ = unlabeled_iter.next()\n        except:\n            unlabeled_iter = iter(test_loader)\n            (x_w, x_s), _ = unlabeled_iter.next()\n        inputs = torch.cat((x_t_w,x_t_s,x_w,x_s)).cuda()\n        optimizer.zero_grad()\n        outputs = net(inputs) \n        y_outputs = outputs[:len(x_t_w)+len(x_t_s)]\n        #print(len(x))\n        y_w , y_s = outputs[len(x_t_w)+len(x_t_s):len(x_t_w)+len(x_t_s)+len(x_w)], outputs[len(x_t_w)+len(x_t_s)+len(x_w):]\n        y=y.cuda()\n        y = torch.cat((y,y)).cuda()\n        lossL = criterion(y_outputs, y)  # Loss\n        pseudo_labels = torch.softmax(y_w,dim=-1)\n        prob,pseudo_labels = torch.max(pseudo_labels,dim=-1)\n        prob_mask = (prob >=threshhold).float()\n        lossU =(criterion(y_s,pseudo_labels) * prob_mask).mean()\n        _, predicted = torch.max(y_outputs.data, 1)\n        _, pseudopredicted = torch.max(y_s,1)\n        pseudopredicted=(pseudopredicted*prob_mask)\n        total += y.size(0)\n        totalpseudo = y_s.size(0)\n        correct += predicted.eq(y.data).cpu().sum() \n        correctpseudo = 0\n        besttrainacc = max((besttrainacc,correct/total))\n        correctpseudo=(pseudopredicted).eq(pseudo_labels*prob_mask).cpu().sum()\n        \n        loss = lossL+ getlambda(epoch, besttrainacc)*lossU\n        loss.backward()  # Backward Propagation\n        optimizer.step() # Optimizer update\n        train_loss += loss.item()\n        if(getlambda(epoch, besttrainacc)==0):\n            correctpseudo=0\n        if batch_idx % log_interval == 0:\n            print('| Epoch [%3d/%3d] Iter[%3d/%3d]\\t\\tLoss: %.4f Acc@1: %.3f%% \\t - \\t pseudoAcc: %.3f%%'\n                %(epoch, num_epochs, batch_idx+1,\n                    (eval_step), loss.item(), 100.*correct/total, 100.*correctpseudo/totalpseudo))\n    acc =test(epoch,net,valid_loader)\n    # Save checkpoint when best model\n    if acc > best_acc:\n        print('| Saving Best model...\\t\\t\\tTop1 = %.2f%%' %(acc))\n        net_save.load_state_dict(net.state_dict(), strict=True)\n        best_acc=acc\n\n","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ssl_net = copy.deepcopy(net_save)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n\ndef test_final(net,testloader):\n    global best_acc\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs_weak, targets) in enumerate(testloader):\n            \n            inputs_weak, targets = inputs_weak.cuda(), targets.cuda()\n            inputs_weak, targets = Variable(inputs_weak), Variable(targets)\n            outputs = net(inputs_weak)\n            loss = criterion(outputs, targets)\n\n            test_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += targets.size(0)\n            correct += predicted.eq(targets.data).cpu().sum()\n            if batch_idx == 0:\n                predicted_concat = predicted.clone()\n            else:\n                predicted_concat = torch.cat((predicted_concat, predicted), 0)\n\n        # Save checkpoint when best model\n    acc = 100.*correct/total\n    print(\"\\n| TEST \\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" %( loss.item(), acc))\n    return predicted_concat.cpu().numpy()\n    \n\npredicted_concat = test_final(ssl_net,final_loader)\n\n\nid_concat =range(len(predicted_concat))\nmy_submission = pd.DataFrame({'Id': id_concat,'Expected': predicted_concat})\n\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission2.csv', index=False)\nprint('we have saved the submission !! ')\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}