{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Imports & Environment","metadata":{}},{"cell_type":"code","source":"from __future__ import print_function, division\nimport os\nimport torch\nfrom skimage import io, transform\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, utils\n\n# Ignore warnings\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport random\nimport collections\nimport shutil\nimport time\nimport glob\nimport csv\nimport torch.backends.cudnn as cudnn\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils.data as data\nimport torchvision.datasets as datasets\nimport torchvision.models as models\nimport torchvision.transforms as transforms\nimport pandas as pd\n\nfrom PIL import Image\nfrom distutils.dir_util import remove_tree\nfrom distutils.dir_util import copy_tree\nfrom torch.utils.data import Dataset\n\nos.chdir('/kaggle/input/')\nfromdirectory = './'\ntodirectory = '/kaggle/working/'\ncopy_tree(fromdirectory, todirectory)\n\nos.chdir('/kaggle/working/fishnet-dataset/Fishnet_Dataset_For_Kaggle_Augmentation_v5/')\nROOT_DIR = os.getcwd()\nDATA_HOME_DIR = ROOT_DIR + ''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Config & Hyperparameters","metadata":{}},{"cell_type":"code","source":"# paths\ndata_path = DATA_HOME_DIR + '/' \nsplit_train_path = data_path + '/train/'\nvalid_path = data_path + '/valid/'\ntest_path = data_path + '/test/'\nsaved_model_path = ROOT_DIR + '/models/'\nsubmission_path = ROOT_DIR + '/submissions/'\n\n# data\nbatch_size = 8\n\n# model\nnb_runs = 1\nnb_aug = 3\n# epochs = 5\nepochs = 5\n# lr = 1e-4\nlr = 1e-4\nclip = 0.001\narchs = [\"resnet152\"]\n\nmodel_names = sorted(name for name in models.__dict__ if name.islower() and not name.startswith(\"__\"))\nbest_prec1 = 0\n\n# model_names = sorted(name for name in models.__dict__)\n# For AlexNet: \n# model = torch.hub.load('pytorch/vision:v0.9.0', 'alexnet', pretrained=True)\n\nbest_prec1 = 0","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Helper Functions for Training","metadata":{}},{"cell_type":"code","source":"def train(train_loader, model, criterion, optimizer, epoch):\n    batch_time = AverageMeter()\n    data_time = AverageMeter()\n    losses = AverageMeter()\n    acc = AverageMeter()\n    end = time.time()\n    \n    # switch to train mode\n    model.train()\n        \n    for i, (images, target) in enumerate(train_loader):\n        # measure data loading time\n        data_time.update(time.time() - end)\n\n        target = target.cuda()\n        image_var = torch.autograd.Variable(images)\n        label_var = torch.autograd.Variable(target)\n        # print (i)\n        # print (\"image_var shape: \", image_var.shape)\n        # print (\"label_var shape: \", label_var.shape)\n        # print (\"Before training\")\n        # compute y_pred\n        y_pred = model(image_var)\n        \n        # print (\"y_pred.shape: \", y_pred.shape)\n        \n        loss = criterion(y_pred, label_var)\n        # print (\"loss: \", loss)\n        \n        # measure accuracy and record loss\n        prec1, prec1 = accuracy(y_pred.data, target, topk=(1, 1))\n        # Change here\n        # prec1 = accuracy_1(y_pred.data, target)\n        \n        losses.update(loss.data, images.size(0))\n        acc.update(prec1, images.size(0))\n\n        # compute gradient and do SGD step\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def validate(val_loader, model, criterion, epoch):\n    batch_time = AverageMeter()\n    losses = AverageMeter()\n    acc = AverageMeter()\n\n    # switch to evaluate mode\n    model.eval()\n\n    end = time.time()\n    for i, (images, labels) in enumerate(val_loader):\n        labels = labels.cuda()\n        image_var = torch.autograd.Variable(images)\n        label_var = torch.autograd.Variable(labels)\n\n        # compute y_pred\n        y_pred = model(image_var)\n        loss = criterion(y_pred, label_var)\n\n        # measure accuracy and record loss\n        prec1, temp_var = accuracy(y_pred.data, labels, topk=(1, 1))\n        # prec1 = accuracy_1(y_pred.data, labels)\n        \n        losses.update(loss.data, images.size(0))\n        acc.update(prec1, images.size(0))\n\n        # measure elapsed time\n        batch_time.update(time.time() - end)\n        end = time.time()\n\n    print('   * EPOCH {epoch} | Accuracy: {acc.avg:.3f} | Loss: {losses.avg:.3f}'.format(epoch=epoch,\n                                                                                         acc=acc,\n                                                                                         losses=losses))\n\n    return acc.avg","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test(test_loader, model, criterion):\n    acc = AverageMeter()\n    losses = AverageMeter()\n    \n    # switch to evaluate mode\n    model.eval()\n    \n    for i, (images, labels) in enumerate(test_loader):\n        print (\"TEST\")\n        labels = labels.cuda()\n        image_var = torch.autograd.Variable(images)\n        label_var = torch.autograd.Variable(labels)\n\n        # compute y_pred\n        y_pred = model(image_var)\n        loss = criterion(y_pred, label_var)\n\n        # measure accuracy and record loss\n        prec1, temp_var = accuracy(y_pred.data, labels, topk=(1, 1))\n        losses.update(loss.data, images.size(0))\n        acc.update(prec1, images.size(0))\n\n    print(' Accuracy:', acc.avg)\n        \n    # for aug in range(nb_aug):\n    #    print(\"   * Predicting on test augmentation {}\".format(aug + 1))\n        \n    '''    for i, (images, filepath) in enumerate(test_loader):\n            # pop extension, treat as id to map\n            filepath = os.path.splitext(os.path.basename(filepath[0]))[0]\n            filepath = int(filepath)\n             \n            print(i, filepath)\n            \n            image_var = torch.autograd.Variable(images)\n            y_pred = model(image_var)\n            # get the index of the max log-probability\n            smax = nn.Softmax()\n            smax_out = smax(y_pred)[0]\n            # cat_prob = smax_out.data[0]\n            # dog_prob = smax_out.data[1]\n            one_prob = smax_out.data[0]\n            two_prob = smax_out.data[1]\n            three_prob = smax_out.data[2]\n            four_prob = smax_out.data[3]\n            five_prob = smax_out.data[4]\n            \n            print(one_prob, two_prob, three_prob, four_prob, five_prob) \n            # prob = dog_prob\n            # if cat_prob > dog_prob:\n            #     prob = 1 - cat_prob\n            # prob = np.around(prob.cpu(), decimals=4)\n            # prob = np.clip(prob, clip, 1-clip)\n            # csv_map[filepath] += (prob / nb_aug)\n\n    sub_fn = submission_path + '{0}epoch_{1}clip_{2}runs'.format(epochs, clip, nb_runs)\n    \n    # for arch in archs:\n    #    sub_fn += \"_{}\".format(arch)\n        \n    # print(\"Writing Predictions to CSV...\")\n    # with open(sub_fn + '.csv', 'w') as csvfile:\n    #    fieldnames = ['id', 'label']\n    #    csv_w = csv.writer(csvfile)\n    #    csv_w.writerow(('id', 'label'))\n    #    for row in sorted(csv_map.items()):\n    #        csv_w.writerow(row)\n   '''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):\n    torch.save(state, filename)\n    if is_best:\n        shutil.copyfile(filename, 'model_best.pth.tar')      ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_learning_rate(optimizer, epoch):\n    \"\"\"Sets the learning rate to the initial LR decayed by 10 every 30 epochs\"\"\"\n    global lr\n    lr = lr * (0.1**(epoch // 30))\n    for param_group in optimizer.state_dict()['param_groups']:\n        param_group['lr'] = lr\n\n\ndef accuracy(y_pred, y_actual, topk=(1, )):\n    \"\"\"Computes the precision@k for the specified values of k\"\"\"\n    maxk = max(topk)\n    batch_size = y_actual.size(0)\n\n    _, pred = y_pred.topk(maxk, 1, True, True)\n    pred = pred.t()\n    correct = pred.eq(y_actual.view(1, -1).expand_as(pred))\n\n    res = []\n    for k in topk:\n        correct_k = correct[:k].view(-1).float().sum(0)\n        res.append(correct_k.mul_(100.0 / batch_size))\n\n    return res\n\ndef accuracy_1(y_pred, y_actual):\n    # print(y_pred.shape)\n    # print(y_actual.shape)\n    \n    return np.sum(y_pred==y_actual)/len(y_actual)*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestImageFolder(data.Dataset):\n    def __init__(self, root, transform=None):\n        images = []\n        for filename in sorted(glob.glob(test_path + \"*.jpg\")):\n            images.append('{}'.format(filename))\n\n        self.root = root\n        self.imgs = images\n        self.transform = transform\n\n    def __getitem__(self, index):\n        filename = self.imgs[index]\n        img = Image.open(os.path.join(self.root, filename))\n        if self.transform is not None:\n            img = self.transform(img)\n        return img, filename\n\n    def __len__(self):\n        return len(self.imgs)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def shear(img):\n    width, height = img.size\n    m = random.uniform(-0.05, 0.05)\n    xshift = abs(m) * width\n    new_width = width + int(round(xshift))\n    img = img.transform((new_width, height), Image.AFFINE,\n                        (1, m, -xshift if m > 0 else 0, 0, 1, 0),\n                        Image.BICUBIC)\n    return img","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Main Training Loop","metadata":{}},{"cell_type":"code","source":"def main(mode=\"train\", resume=False):\n    \n    global best_prec1\n    \n    for arch in archs:\n\n        # create model\n        print(\"=> Starting {0} on '{1}' model\".format(mode, arch))\n        model = models.__dict__[arch](pretrained=True)\n        # Don't update non-classifier learned features in the pretrained networks\n        for param in model.parameters():\n            param.requires_grad = False\n        # Replace the last fully-connected layer\n        # Parameters of newly constructed modules have requires_grad=True by default\n        # Final dense layer needs to replaced with the previous out chans, and number of classes\n        # in this case -- resnet 101 - it's 2048 with two classes (cats and dogs)\n        # model.fc = nn.Linear(2048, 2)\n        # in this case -- resnet 101 - it's 2048 with five classes, first change here. \n        # model.fc == nn.Linear(2048, 5)\n        model.fc = nn.Sequential(nn.Linear(2048, 512),\n                                 nn.ReLU(),\n                                 nn.Dropout(0.2),\n                                 nn.Linear(512, 5),\n                                 nn.LogSoftmax(dim=1))\n        \n        if arch.startswith('alexnet') or arch.startswith('vgg'):\n            model.features = torch.nn.DataParallel(model.features)\n            model.cuda()\n        else:\n            model = torch.nn.DataParallel(model).cuda()\n            \n        # optionally resume from a checkpoint\n        if resume:\n            if os.path.isfile(resume):\n                print(\"=> Loading checkpoint '{}'\".format(resume))\n                checkpoint = torch.load(resume)\n                start_epoch = checkpoint['epoch']\n                best_prec1 = checkpoint['best_prec1']\n                model.load_state_dict(checkpoint['state_dict'])\n                print(\"=> Loaded checkpoint (epoch {})\".format(checkpoint['epoch']))\n            else:\n                print(\"=> No checkpoint found at '{}'\".format(args.resume))\n\n        cudnn.benchmark = True\n\n        # Data loading code\n        traindir = split_train_path\n        valdir = valid_path\n        testdir = test_path\n\n        normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n\n        train_loader = data.DataLoader(\n            datasets.ImageFolder(traindir,\n                                 transforms.Compose([\n                                     # transforms.Lambda(shear),\n                                     transforms.RandomResizedCrop(224),\n                                     transforms.RandomHorizontalFlip(),\n                                     transforms.ToTensor(),\n                                     normalize,\n                                 ])),\n            batch_size=batch_size,\n            shuffle=True,\n            num_workers=4,\n            pin_memory=True)\n\n        val_loader = data.DataLoader(\n            datasets.ImageFolder(valdir,\n                                 transforms.Compose([\n                                     transforms.Resize(256),\n                                     transforms.CenterCrop(224),\n                                     transforms.ToTensor(),\n                                     normalize,\n                                 ])),\n            batch_size=batch_size,\n            shuffle=True,\n            num_workers=4,\n            pin_memory=True)\n\n        test_loader = data.DataLoader(\n            TestImageFolder(testdir,\n                            transforms.Compose([\n                                # transforms.Lambda(shear),\n                                transforms.Resize(256),\n                                transforms.CenterCrop(224),\n                                transforms.ToTensor(),\n                                normalize,\n                            ])),\n            batch_size=1,\n            shuffle=False,\n            num_workers=1,\n            pin_memory=False)\n        \n        criterion = nn.CrossEntropyLoss().cuda()\n        \n        if mode == \"test\":\n            test(test_loader, model, criterion)\n            return\n        \n        # define loss function (criterion) and pptimizer\n        criterion = nn.CrossEntropyLoss().cuda()\n        \n        if mode == \"validate\":\n            validate(val_loader, model, criterion, 0)\n            return\n\n        optimizer = optim.Adam(model.module.fc.parameters(), lr, weight_decay=1e-4)\n\n        for epoch in range(epochs):\n            adjust_learning_rate(optimizer, epoch)\n\n            # train for one epoch\n            train(train_loader, model, criterion, optimizer, epoch)\n\n            # evaluate on validation set\n            prec1 = validate(val_loader, model, criterion, epoch)\n\n            # remember best Accuracy and save checkpoint\n            is_best = prec1 > best_prec1\n            best_prec1 = max(prec1, best_prec1)\n            save_checkpoint({\n                'epoch': epoch + 1,\n                'arch': arch,\n                'state_dict': model.state_dict(),\n                'best_prec1': best_prec1,\n            }, is_best)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Run Train","metadata":{}},{"cell_type":"code","source":"main(mode=\"train\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"main(mode=\"validate\", resume='model_best.pth.tar')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Run Test","metadata":{}},{"cell_type":"code","source":"# os.mkdir('/kaggle/working/submissions')\nmain(mode=\"test\", resume='model_best.pth.tar')","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}