{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport os\nimport glob\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.nn.init as init\nimport torchvision.models as models\nfrom torchvision import transforms\nimport torchvision.transforms as transforms\nimport torch.utils.data as data\nimport torchvision\nimport random\nimport math\nfrom torch.autograd import Variable\n#import matplotlib.pyplot as plt\n#from modules import *\n#import torchvision.models as models_pytorch\n#import h5py\n#import torch.optim as optim\n#import augmentations\nfrom torch.nn.functional import kl_div, softmax, log_softmax\nimport torch.optim as optim\nimport torch.utils.data as data\nfrom os.path import exists, join, split\nfrom os import listdir\nfrom os.path import join\nfrom PIL import Image\nimport random\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_class = 10 #number of classes\n\nseed=111 #seed for the algorithm\nbatch_size = 32\nnum_train =100 # number of training image by classe\ncutout=16  # parameter for the cutout\nnum_epochs=200\n#Validation set size\nvalid_size = 200\nlr=0.1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## First let us define a CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nbn_momentum = 0.9\n\n\ndef conv3x3(in_planes, out_planes, stride=1):\n    return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=True)\n\n\ndef conv_init(m):\n    classname = m.__class__.__name__\n    if classname.find('Conv') != -1:\n        init.xavier_uniform_(m.weight, gain=np.sqrt(2))\n        init.constant_(m.bias, 0)\n    elif classname.find('BatchNorm') != -1:\n        init.constant_(m.weight, 1)\n        init.constant_(m.bias, 0)\n\n\nclass WideBasic(nn.Module):\n    def __init__(self, in_planes, planes, dropout_rate, stride=1):\n        super(WideBasic, self).__init__()\n        self.bn1 = nn.BatchNorm2d(in_planes, momentum=bn_momentum)\n        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, padding=1, bias=True)\n        self.dropout = nn.Dropout(p=dropout_rate)\n        self.bn2 = nn.BatchNorm2d(planes, momentum=bn_momentum)\n        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=True)\n\n        self.shortcut = nn.Sequential()\n        if stride != 1 or in_planes != planes:\n            self.shortcut = nn.Sequential(\n                nn.Conv2d(in_planes, planes, kernel_size=1, stride=stride, bias=True),\n            )\n\n    def forward(self, x):\n        out = self.dropout(self.conv1(F.relu(self.bn1(x))))\n        out = self.conv2(F.relu(self.bn2(out)))\n        out += self.shortcut(x)\n\n        return out\n\n\nclass WideResNet(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out = self.linear(out)\n\n        return out\n\nclass WideResNet_NDA(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_NDA, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        logit = self.linear(out)\n\n        return out,logit\n\n\n\n\nclass WideResNet_conf(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_conf, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.linear = nn.Linear(nStages[3], num_classes)\n        self.fconfid =  nn.Linear(nStages[3], 1)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.layer2(out)\n        out = self.layer3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out_final = self.linear(out)\n        out_conf = self.fconfid(out)\n\n        return out_final,out_conf\n\n\nclass WideResNet_drop(nn.Module):\n    def __init__(self, depth, widen_factor, dropout_rate, num_classes):\n        super(WideResNet_drop, self).__init__()\n        self.in_planes = 16\n\n        assert ((depth - 4) % 6 == 0), 'Wide-resnet depth should be 6n+4'\n        n = int((depth - 4) / 6)\n        k = widen_factor\n\n        nStages = [16, 16*k, 32*k, 64*k]\n\n        self.conv1 = conv3x3(3, nStages[0])\n        self.layer1 = self._wide_layer(WideBasic, nStages[1], n, dropout_rate, stride=1)\n        self.dropout1 = nn.Dropout2d(0.25)\n        self.layer2 = self._wide_layer(WideBasic, nStages[2], n, dropout_rate, stride=2)\n        self.dropout2 = nn.Dropout2d(0.25)\n        self.layer3 = self._wide_layer(WideBasic, nStages[3], n, dropout_rate, stride=2)\n        self.dropout3 = nn.Dropout2d(0.25)\n        self.bn1 = nn.BatchNorm2d(nStages[3], momentum=bn_momentum)\n        self.dropout4 = nn.Dropout2d(0.25)\n        self.linear = nn.Linear(nStages[3], num_classes)\n\n        # self.apply(conv_init)\n\n    def _wide_layer(self, block, planes, num_blocks, dropout_rate, stride):\n        strides = [stride] + [1]*(num_blocks-1)\n        layers = []\n\n        for stride in strides:\n            layers.append(block(self.in_planes, planes, dropout_rate, stride))\n            self.in_planes = planes\n\n        return nn.Sequential(*layers)\n\n    def forward(self, x):\n        out = self.conv1(x)\n        out = self.layer1(out)\n        out = self.dropout1(out)\n        out = self.layer2(out)\n        out = self.dropout2(out)\n        out = self.layer3(out)\n        out = self.dropout3(out)\n        out = F.relu(self.bn1(out))\n        # out = F.avg_pool2d(out, 8)\n        out = F.adaptive_avg_pool2d(out, (1, 1))\n        out = out.view(out.size(0), -1)\n        out = self.dropout4(out)\n        out = self.linear(out)\n\n        return out","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## now let us define a dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Dataset_sub_CIFAR(data.Dataset):\n\n    def __init__(self, data_feature, data_target,transform,phase='label'):\n        self.data_feature = data_feature\n        self.data_target = data_target\n        self.transform = transform\n        self.phase=phase\n\n\n\n    def __len__(self):\n        return len(self.data_feature)\n\n    def __getitem__(self, index):\n        # load image as ndarray type (Height * Width * Channels)\n        # be carefull for converting dtype to np.uint8 [Unsigned integer (0 to 255)]\n        # in this example, i don't use ToTensor() method of torchvision.transforms\n        # so you can convert numpy ndarray shape to tensor in PyTorch (H, W, C) --> (C, H, W)\n        if self.phase=='label':\n            data_feature = self.transform(Image.fromarray(np.uint8(self.data_feature[index])))\n            data_target =  self.data_target[index]\n            return data_feature, data_target\n\n        else:\n            data_feature = self.data_feature[index].float()\n            return data_feature\n\n\nclass CutoutDefault(object):\n    \"\"\"\n    Reference : https://github.com/quark0/darts/blob/master/cnn/utils.py\n    \"\"\"\n    def __init__(self, length):\n        self.length = length\n\n    def __call__(self, img):\n        if self.length <= 0:\n            return img\n        h, w = img.size(1), img.size(2)\n        mask = np.ones((h, w), np.float32)\n        y = np.random.randint(h)\n        x = np.random.randint(w)\n\n        y1 = np.clip(y - self.length // 2, 0, h)\n        y2 = np.clip(y + self.length // 2, 0, h)\n        x1 = np.clip(x - self.length // 2, 0, w)\n        x2 = np.clip(x + self.length // 2, 0, w)\n\n        mask[y1: y2, x1: x2] = 0.\n        mask = torch.from_numpy(mask)\n        mask = mask.expand_as(img)\n        img *= mask\n        return img\n\n    \ntransform_train = transforms.Compose([\n    transforms.RandomCrop(32, padding=4, padding_mode = 'reflect'),\n    transforms.RandomHorizontalFlip(),\n    transforms.ToTensor(),\n    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),\n    CutoutDefault(cutout),\n])\n\ntransform_test = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)),\n])\n\n\n\n#Dataset loading\nCIFAR10_train_dataset = torchvision.datasets.CIFAR10(root='./data', train=True, transform=None, download=True)\nCIFAR10_test_dataset = torchvision.datasets.CIFAR10(root='./data', train=False, transform=None, download=True)\nnp.random.seed(seed=seed)\npermuation=np.random.permutation(len(CIFAR10_train_dataset.targets))\n\nOriginal_train_data_x = (CIFAR10_train_dataset.data)\nOriginal_train_data_y = np.array(CIFAR10_train_dataset.targets)\nOriginal_train_data_x = Original_train_data_x[permuation]\nOriginal_train_data_y = Original_train_data_y[permuation]\n\nOriginal_test_data_x = CIFAR10_test_dataset.data\nOriginal_test_data_y = np.array(CIFAR10_test_dataset.targets)\n\n\n\n#Selection of 250 labeled images for training and 2000 for validation\nincr_class = torch.zeros(num_class)\ntrain_idx_dico = {} #labeled images index dictionnary\n\nfor i in range(num_class):\n    train_idx_dico[str(i)] = []\n\nvalid_idx = np.zeros(num_class * valid_size, dtype=np.int32) #validation images indexes (2000)\nincr_t = 0\nincr_v = 0\nincrtotal = 0\n\nfor idx in range(len(Original_train_data_y)):\n    class_y = Original_train_data_y[idx]\n    incrtotal += 1\n\n    train_idx_dico[str(class_y)].append(idx)\n    incr_class[class_y] += 1 #count the number of image per class\n    incr_t += 1\n\n\ntrain_idx = np.zeros(num_class * num_train, dtype=np.int32) #train labeled images indexes (250)\nlist_train_id = []\nlist_unalabel_id = []\nvalid_idx = []\nunlabel_idx_dico = {}\nfor i in range(num_class):\n    unlabel_idx_dico[str(i)] = []\nfor i in range(num_class):\n    list_train_id = list_train_id + train_idx_dico[str(i)][0:num_train]\n    valid_idx =valid_idx + train_idx_dico[str(i)][num_train:num_train+valid_size]\n    list_unalabel_id = list_unalabel_id + train_idx_dico[str(i)][num_train+valid_size::]\n    unlabel_idx_dico[str(i)] = train_idx_dico[str(i)][num_train::]\n\n#Get labeled and unlabeled data\n\nx_train = Original_train_data_x[[int(i) for i in list_train_id]]\ny_train = Original_train_data_y[[int(i) for i in list_train_id]]\n\nx_unlabeled = Original_train_data_x[[int(i) for i in list_unalabel_id]]\ny_unlabeled = Original_train_data_y[[int(i) for i in list_unalabel_id]]\n\n#Get validation set data\nx_valid = Original_train_data_x[[int(i) for i in valid_idx]]\ny_valid = Original_train_data_y[[int(i) for i in valid_idx]]\n\n# Printing the size of the training, validation and test sets\nprint('Number of training examples: ' + str(x_train.shape[0]))\nprint('Number of unlabeled examples: ' + str(x_unlabeled.shape[0]))\nprint('Number of validation examples: ' + str(x_valid.shape[0]))\n\nclasses = ('plane', 'car', 'bird', 'cat',\n           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n\n#Dataloader creation\n\ntest_loader = torch.utils.data.DataLoader(\n    Dataset_sub_CIFAR(Original_test_data_x, Original_test_data_y, transform=transform_test),\n    batch_size = batch_size,\n    shuffle=False, num_workers=2)\n\ntrain_loader = torch.utils.data.DataLoader(\n    Dataset_sub_CIFAR(x_train, y_train, transform=transform_train),\n    batch_size=batch_size,shuffle=True, num_workers=2) #num_workers = 2 ou 1\n\nvalid_loader = torch.utils.data.DataLoader(\n    Dataset_sub_CIFAR(x_valid, y_valid, transform=transform_test),\n    batch_size=batch_size,\n    shuffle=False, num_workers=2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now we build the CNN and the optimizer\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\n#Networks creation\nnet = WideResNet(28, 2, dropout_rate=0.0, num_classes=num_class)\nnet =net.cuda()\nnet_save = WideResNet(28, 2, dropout_rate=0.0, num_classes=num_class) # model where to save the results\nnet_save =net_save.cuda()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def learning_rate_scheduler(init, epoch):\n    optim_factor = 0\n    if(epoch > 200):\n        optim_factor = 3\n    elif(epoch > 160):\n        optim_factor = 2\n    elif(epoch > 80):\n        optim_factor = 1\n\n    return init*math.pow(0.1, optim_factor)\n\n\n# Training\ndef train(epoch,net,trainloader,log_interval=15):\n    net.train()\n    train_loss = 0\n    correct = 0\n    total = 0\n    print('\\n=> Training Epoch #%d, LR=%.4f' %(epoch, learning_rate_scheduler(lr, epoch)))\n    optimizer = optim.SGD(net.parameters(), lr=learning_rate_scheduler(lr, epoch), momentum=0.9, weight_decay=5e-4)\n    for batch_idx, (inputs, targets) in enumerate(trainloader):\n \n        inputs, targets = inputs.cuda(), targets.cuda() # GPU settings\n        optimizer.zero_grad()\n        inputs, targets = Variable(inputs), Variable(targets)\n        outputs = net(inputs)               # Forward Propagation\n        loss = criterion(outputs, targets)  # Loss\n        loss.backward()  # Backward Propagation\n        optimizer.step() # Optimizer update\n\n        train_loss += loss.item()\n        _, predicted = torch.max(outputs.data, 1)\n        total += targets.size(0)\n        correct += predicted.eq(targets.data).cpu().sum()\n        if batch_idx % log_interval == 0:\n            print('| Epoch [%3d/%3d] Iter[%3d/%3d]\\t\\tLoss: %.4f Acc@1: %.3f%%'\n                %(epoch, num_epochs, batch_idx+1,\n                    (len(trainloader.dataset)//batch_size)+1, loss.item(), 100.*correct/total))\n\n\ndef test(epoch,net,testloader):\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(testloader):\n            \n            inputs, targets = inputs.cuda(), targets.cuda()\n            inputs, targets = Variable(inputs), Variable(targets)\n            outputs = net(inputs)\n            loss = criterion(outputs, targets)\n\n            test_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += targets.size(0)\n            correct += predicted.eq(targets.data).cpu().sum()\n\n        \n    acc = 100.*correct/total\n    print(\"\\n| Validation Epoch #%d\\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" %(epoch, loss.item(), acc))\n    return acc\n\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_acc=0\nfor epoch in range(num_epochs):\n\n\n    train(epoch,net,train_loader)\n    acc =test(epoch,net,valid_loader)\n    # Save checkpoint when best model\n    if acc > best_acc:\n        print('| Saving Best model...\\t\\t\\tTop1 = %.2f%%' %(acc))\n        net_save.load_state_dict(net.state_dict(), strict=True)\n        best_acc=acc\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\n\n\ndef test_final(net,testloader):\n    global best_acc\n    net.eval()\n    test_loss = 0\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for batch_idx, (inputs, targets) in enumerate(testloader):\n            \n            inputs, targets = inputs.cuda(), targets.cuda()\n            inputs, targets = Variable(inputs), Variable(targets)\n            outputs = net(inputs)\n            loss = criterion(outputs, targets)\n\n            test_loss += loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total += targets.size(0)\n            correct += predicted.eq(targets.data).cpu().sum()\n            if batch_idx == 0:\n                predicted_concat = predicted.clone()\n            else:\n                predicted_concat = torch.cat((predicted_concat, predicted), 0)\n\n        # Save checkpoint when best model\n    acc = 100.*correct/total\n    print(\"\\n| TEST \\t\\t\\tLoss: %.4f Acc@1: %.2f%%\" %( loss.item(), acc))\n    return predicted_concat.cpu().numpy()\n    \n\npredicted_concat = test_final(net,test_loader)\n\n\nid_concat =range(len(predicted_concat))\nmy_submission = pd.DataFrame({'Id': id_concat,'Expected': predicted_concat})\n\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission2.csv', index=False)\nprint('we have saved the submission !! ')\n    ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}