{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import Dataset, random_split, DataLoader\nfrom torchvision.datasets import ImageFolder\nfrom PIL import Image\nimport torchvision\nimport torchvision.models as models\nimport matplotlib.pyplot as plt\nimport torchvision.transforms as transforms\nfrom sklearn.metrics import f1_score\nimport torch.nn.functional as F\nimport torch.nn as nn\nfrom torchvision.utils import make_grid\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# extract files\nimport zipfile\n\n# train data\nif 'train' not in os.listdir('.'):\n    with zipfile.ZipFile(\"../input/dogs-vs-cats/train.zip\",\"r\") as z:\n        z.extractall(\".\")\n\n# test data\nif 'test1' not in os.listdir('.'):\n    with zipfile.ZipFile(\"../input/dogs-vs-cats/test1.zip\",\"r\") as z:\n        z.extractall(\".\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_DIR = '/kaggle/working' \n\nTRAIN_DIR = DATA_DIR + '/train'                           # Contains training images\nTEST_DIR = DATA_DIR + '/test1'                             # Contains test images\n\nTRAIN_CSV = DATA_DIR + '/train.csv'                       # Contains real labels for training images\nTEST_CSV = './jovian-pytorch-z2g/submission.csv'   # Contains dummy labels for test image\n\nfiles = os.listdir(TRAIN_DIR)\nfiles[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_dog_cat(fname):\n    parts = fname.split('.')\n    return ' '.join(parts[:-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"parse_dog_cat(files[4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def open_image(path):\n    with open(path, 'rb') as f:\n        img = Image.open(f)\n        return img.convert('RGB')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DogCatDataset(Dataset):\n    def __init__(self, root, transform):\n        super().__init__()\n        self.root = root\n        self.files = [fname for fname in os.listdir(root) if fname.endswith('.jpg')]\n        self.classes = list(set(parse_dog_cat(fname) for fname in files))\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.files)\n\n    def __getitem__(self, i):\n        fname = self.files[i]\n        fpath = os.path.join(self.root, fname)\n        img = self.transform(open_image(fpath))\n        class_idx = self.classes.index(parse_dog_cat(fname))\n        return img, class_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torchvision.transforms as T\nimg_size = 224\nimagenet_stats = ([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\ndataset = DogCatDataset(TRAIN_DIR, T.Compose([T.Resize(img_size), \n                                           T.Pad(8, padding_mode='reflect'),\n                                           T.RandomCrop(img_size), \n                                           T.ToTensor(), \n                                           T.Normalize(*imagenet_stats)]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def denormalize(images, means, stds):\n    if len(images.shape) == 3:\n        images = images.unsqueeze(0)\n    means = torch.tensor(means).reshape(1, 3, 1, 1)\n    stds = torch.tensor(stds).reshape(1, 3, 1, 1)\n    return images * stds + means\n\ndef show_image(img_tensor, label):\n    print('Label:', dataset.classes[label], '(' + str(label) + ')')\n    img_tensor = denormalize(img_tensor, *imagenet_stats)[0].permute((1, 2, 0))\n    plt.imshow(img_tensor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_image(*dataset[2])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Creating Training and Validation Sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"val_pct = 0.1 # validation percentage\nval_size = int(val_pct * len(dataset))\n\ntrain_ds, valid_ds = random_split(dataset, [len(dataset) - val_size, val_size])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 256\n\ntrain_dl = DataLoader(train_ds, batch_size, shuffle=True, num_workers=4, pin_memory=True)\nvalid_dl = DataLoader(valid_ds, batch_size*2, num_workers=4, pin_memory=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_batch(dl):\n    for images, labels in dl:\n        fig, ax = plt.subplots(figsize=(16, 16))\n        ax.set_xticks([]); ax.set_yticks([])\n        images = denormalize(images[:64], *imagenet_stats)\n        ax.imshow(make_grid(images, nrow=8).permute(1, 2, 0))\n        break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_batch(train_dl)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modifying a Pretrained Model (ResNet34)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def accuracy(outputs, labels):\n    _, preds = torch.max(outputs, dim=1)\n    return torch.tensor(torch.sum(preds == labels).item() / len(preds))\n\n\nclass ImageClassificationBase(nn.Module):\n    def training_step(self, batch):\n        images, labels = batch\n        out = self(images)                  # Generate predictions\n        loss = F.cross_entropy(out, labels)  # Calculate loss\n        return loss\n\n    def validation_step(self, batch):\n        images, labels = batch\n        out = self(images)                    # Generate predictions\n        loss = F.cross_entropy(out, labels)   # Calculate loss\n        acc = accuracy(out, labels)           # Calculate accuracy\n        return {'val_loss': loss.detach(), 'val_acc': acc}\n\n    def validation_epoch_end(self, outputs):\n        batch_losses = [x['val_loss'] for x in outputs]\n        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses\n        batch_accs = [x['val_acc'] for x in outputs]\n        epoch_acc = torch.stack(batch_accs).mean()      # Combine accuracies\n        return {'val_loss': epoch_loss.item(), 'val_acc': epoch_acc.item()}\n\n    def epoch_end(self, epoch, result):\n        print(\"Epoch [{}],{} train_loss: {:.4f}, val_loss: {:.4f}, val_acc: {:.4f}\".format(\n            epoch, \"last_lr: {:.5f},\".format(result['lrs'][-1]) if 'lrs' in result else '', \n            result['train_loss'], result['val_loss'], result['val_acc']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PetsModel(ImageClassificationBase):\n    def __init__(self, num_classes, pretrained=True):\n        super().__init__()\n        # Use a pretrained model\n        self.network = models.resnet34(pretrained=pretrained)\n        # Replace last layer\n        self.network.fc = nn.Linear(self.network.fc.in_features, num_classes)\n\n    def forward(self, xb):\n        return self.network(xb)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GPU Utilities and Training Loop"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_default_device():\n    \"\"\"Pick GPU if available, else CPU\"\"\"\n    if torch.cuda.is_available():\n        return torch.device('cuda')\n    else:\n        return torch.device('cpu')\n\n\ndef to_device(data, device):\n    \"\"\"Move tensor(s) to chosen device\"\"\"\n    if isinstance(data, (list, tuple)):\n        return [to_device(x, device) for x in data]\n    return data.to(device, non_blocking=True)\n\n\nclass DeviceDataLoader():\n    \"\"\"Wrap a dataloader to move data to a device\"\"\"\n\n    def __init__(self, dl, device):\n        self.dl = dl\n        self.device = device\n\n    def __iter__(self):\n        \"\"\"Yield a batch of data after moving it to device\"\"\"\n        for b in self.dl:\n            yield to_device(b, self.device)\n\n    def __len__(self):\n        \"\"\"Number of batches\"\"\"\n        return len(self.dl)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm.notebook import tqdm\n\n@torch.no_grad()\ndef evaluate(model, val_loader):\n    model.eval()\n    outputs = [model.validation_step(batch) for batch in val_loader]\n    return model.validation_epoch_end(outputs)\n\n\ndef fit(epochs, lr, model, train_loader, val_loader, opt_func=torch.optim.SGD):\n    history = []\n    optimizer = opt_func(model.parameters(), lr)\n    for epoch in range(epochs):\n        # Training Phase\n        model.train()\n        train_losses = []\n        for batch in tqdm(train_loader):\n            loss = model.training_step(batch)\n            train_losses.append(loss)\n            loss.backward()\n            optimizer.step()\n            optimizer.zero_grad()\n        # Validation phase\n        result = evaluate(model, val_loader)\n        result['train_loss'] = torch.stack(train_losses).mean().item()\n        model.epoch_end(epoch, result)\n        history.append(result)\n    return history\n\ndef get_lr(optimizer):\n    for param_group in optimizer.param_groups:\n        return param_group['lr']\n\ndef fit_one_cycle(epochs, max_lr, model, train_loader, val_loader,\n                  weight_decay=0, grad_clip=None, opt_func=torch.optim.SGD):\n    torch.cuda.empty_cache()\n    history = []\n\n    # Set up custom optimizer with weight decay\n    optimizer = opt_func(model.parameters(), max_lr, weight_decay=weight_decay)\n    # Set up one-cycle learning rate scheduler\n    sched = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr, epochs=epochs,\n                                                steps_per_epoch=len(train_loader))\n\n    for epoch in range(epochs):\n        # Training Phase\n        model.train()\n        train_losses = []\n        lrs = []\n        for batch in tqdm(train_loader):\n            loss = model.training_step(batch)\n            train_losses.append(loss)\n            loss.backward()\n\n            # Gradient clipping\n            if grad_clip:\n                nn.utils.clip_grad_value_(model.parameters(), grad_clip)\n\n            optimizer.step()\n            optimizer.zero_grad()\n\n            # Record & update learning rate\n            lrs.append(get_lr(optimizer))\n            sched.step()\n\n        # Validation phase\n        result = evaluate(model, val_loader)\n        result['train_loss'] = torch.stack(train_losses).mean().item()\n        result['lrs'] = lrs\n        model.epoch_end(epoch, result)\n        history.append(result)\n    return history","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = get_default_device()\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dl = DeviceDataLoader(train_dl, device)\nvalid_dl = DeviceDataLoader(valid_dl, device)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Finetuning the Pretrained Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = PetsModel(len(dataset.classes))\nto_device(model, device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = [evaluate(model, valid_dl)]\nhistory","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 6\nmax_lr = 0.01\ngrad_clip = 0.1\nweight_decay = 1e-4\nopt_func = torch.optim.Adam","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory += fit_one_cycle(epochs, max_lr, model, train_dl, valid_dl, \n                         grad_clip=grad_clip, \n                         weight_decay=weight_decay, \n                         opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training a model from scratch\n\nLet's repeat the training without using weights from the pretrained ResNet34 model."},{"metadata":{"trusted":true},"cell_type":"code","source":"model2 = PetsModel(len(dataset.classes), pretrained=False)\nto_device(model2, device);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history2 = [evaluate(model2, valid_dl)]\nhistory2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nhistory2 += fit_one_cycle(epochs, max_lr, model2, train_dl, valid_dl, \n                         grad_clip=grad_clip, \n                         weight_decay=weight_decay, \n                         opt_func=opt_func)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install jovian --upgrade --quiet","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import jovian","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jovian.commit(project='transfer-learning-pytorch')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jovian.submit(assignment=\"zerotogans-project\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}