{"nbformat_minor": 1, "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"name": "python", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "mimetype": "text/x-python", "version": "3.6.3", "file_extension": ".py"}}, "nbformat": 4, "cells": [{"cell_type": "markdown", "metadata": {"collapsed": true, "_uuid": "11db94b40434802ac10d5ed76e21d33b9ff6befe", "_cell_guid": "f5af2eca-5b02-4d89-a36d-8ca330851ac3"}, "source": ["# Transfer learning in kernels with PyTorch\n", "\n", "Following the same strategy from Beluga's kernel [Use pretrained Keras models](https://www.kaggle.com/gaborfodor/use-pretrained-keras-models-lb-0-3), this kernel uses a dataset with PyTorch pretrained networks weights. \n", "\n", "Training in the CPU is quite slow, but it is still feasible to use a pre-trained network, replace the final layer and train just this last layer. \n", "\n", "Thanks Beluga for your great kernel. This one uses not only the concept but also a lot of the code. "]}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["%matplotlib inline\n", "import time\n", "import numpy as np\n", "import pandas as pd\n", "import datetime as dt\n", "import matplotlib.pyplot as plt\n", "from mpl_toolkits.axes_grid1 import ImageGrid\n", "from os import listdir, makedirs, getcwd, remove\n", "from os.path import isfile, join, abspath, exists, isdir, expanduser\n", "from PIL import Image\n", "import torch\n", "from torch.optim import lr_scheduler\n", "from torch.autograd import Variable\n", "from torch.utils.data import Dataset, DataLoader\n", "import torchvision\n", "from torchvision import transforms, datasets, models"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["np.random.seed(0)"], "outputs": []}, {"cell_type": "markdown", "metadata": {}, "source": ["This [dataset](https://www.kaggle.com/pvlima/pretrained-pytorch-models) has the PyTorch weights for some pre-trained networks.\n", "\n", "We have to copy the pretrained models to the cache directory (~/.torch/models) where PyTorch is looking for them."]}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["!ls ../input/pretrained-pytorch-models/"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["cache_dir = expanduser(join('~', '.torch'))\n", "if not exists(cache_dir):\n", "    makedirs(cache_dir)\n", "models_dir = join(cache_dir, 'models')\n", "if not exists(models_dir):\n", "    makedirs(models_dir)"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["!cp ../input/pretrained-pytorch-models/* ~/.torch/models/"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["!ls ~/.torch/models"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["!ls ../input/dog-breed-identification"], "outputs": []}, {"cell_type": "markdown", "metadata": {}, "source": ["Using just 16 most frequent breeds to keep the running time under the kernel limit"]}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["INPUT_SIZE = 224\n", "NUM_CLASSES = 16\n", "data_dir = '../input/dog-breed-identification/'\n", "labels = pd.read_csv(join(data_dir, 'labels.csv'))\n", "sample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\n", "print(len(listdir(join(data_dir, 'train'))), len(labels))\n", "print(len(listdir(join(data_dir, 'test'))), len(sample_submission))"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\n", "labels = labels[labels['breed'].isin(selected_breed_list)]\n", "labels['target'] = 1\n", "labels['rank'] = labels.groupby('breed').rank()['id']\n", "labels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\n", "\n", "train = labels_pivot.sample(frac=0.8)\n", "valid = labels_pivot[~labels_pivot['id'].isin(train['id'])]\n", "print(train.shape, valid.shape)"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["class DogsDataset(Dataset):\n", "    def __init__(self, labels, root_dir, subset=False, transform=None):\n", "        self.labels = labels\n", "        self.root_dir = root_dir\n", "        self.transform = transform\n", "    \n", "    def __len__(self):\n", "        return len(self.labels)\n", "    \n", "    def __getitem__(self, idx):\n", "        img_name = '{}.jpg'.format(self.labels.iloc[idx, 0])\n", "        fullname = join(self.root_dir, img_name)\n", "        image = Image.open(fullname)\n", "        labels = self.labels.iloc[idx, 1:].as_matrix().astype('float')\n", "        labels = np.argmax(labels)\n", "        if self.transform:\n", "            image = self.transform(image)\n", "        return [image, labels]"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["normalize = transforms.Normalize(\n", "   mean=[0.485, 0.456, 0.406],\n", "   std=[0.229, 0.224, 0.225]\n", ")\n", "ds_trans = transforms.Compose([transforms.Scale(224),\n", "                               transforms.CenterCrop(224),\n", "                               transforms.ToTensor(),\n", "                               normalize])\n", "train_ds = DogsDataset(train, data_dir+'train/', transform=ds_trans)\n", "valid_ds = DogsDataset(valid, data_dir+'train/', transform=ds_trans)\n", "\n", "train_dl = DataLoader(train_ds, batch_size=4, shuffle=True, num_workers=4)\n", "valid_dl = DataLoader(valid_ds, batch_size=4, shuffle=True, num_workers=4)"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["def imshow(axis, inp):\n", "    \"\"\"Denormalize and show\"\"\"\n", "    inp = inp.numpy().transpose((1, 2, 0))\n", "    mean = np.array([0.485, 0.456, 0.406])\n", "    std = np.array([0.229, 0.224, 0.225])\n", "    inp = std * inp + mean\n", "    axis.imshow(inp)"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["img, label = next(iter(train_dl))\n", "print(img.size(), label.size())\n", "fig = plt.figure(1, figsize=(16, 4))\n", "grid = ImageGrid(fig, 111, nrows_ncols=(1, 4), axes_pad=0.05)    \n", "for i in range(img.size()[0]):\n", "    ax = grid[i]\n", "    imshow(ax, img[i])"], "outputs": []}, {"cell_type": "markdown", "metadata": {}, "source": ["# ResNet50\n", "\n", "### Just try the model "]}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["use_gpu = torch.cuda.is_available()\n", "resnet = models.resnet50(pretrained=True)\n", "inputs, labels = next(iter(train_dl))\n", "if use_gpu:\n", "    resnet = resnet.cuda()\n", "    inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())   \n", "else:\n", "    inputs, labels = Variable(inputs), Variable(labels)\n", "outputs = resnet(inputs)\n", "outputs.size()"], "outputs": []}, {"cell_type": "markdown", "metadata": {}, "source": ["The model seems to work OK. Resnet outputs probabilities for the imagenet 1000 labels as expected. "]}, {"cell_type": "markdown", "metadata": {}, "source": ["### Replace last layer and train\n", "\n", "Will replace the last layer with one that predicts the 16 classes. The network weights will be fixed expected for the last layer that is trained."]}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["def train_model(dataloders, model, criterion, optimizer, scheduler, num_epochs=25):\n", "    since = time.time()\n", "    use_gpu = torch.cuda.is_available()\n", "    best_model_wts = model.state_dict()\n", "    best_acc = 0.0\n", "    dataset_sizes = {'train': len(dataloders['train'].dataset), \n", "                     'valid': len(dataloders['valid'].dataset)}\n", "\n", "    for epoch in range(num_epochs):\n", "        for phase in ['train', 'valid']:\n", "            if phase == 'train':\n", "                scheduler.step()\n", "                model.train(True)\n", "            else:\n", "                model.train(False)\n", "\n", "            running_loss = 0.0\n", "            running_corrects = 0\n", "\n", "            for inputs, labels in dataloders[phase]:\n", "                if use_gpu:\n", "                    inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n", "                else:\n", "                    inputs, labels = Variable(inputs), Variable(labels)\n", "\n", "                optimizer.zero_grad()\n", "\n", "                outputs = model(inputs)\n", "                _, preds = torch.max(outputs.data, 1)\n", "                loss = criterion(outputs, labels)\n", "\n", "                if phase == 'train':\n", "                    loss.backward()\n", "                    optimizer.step()\n", "\n", "                running_loss += loss.data[0]\n", "                running_corrects += torch.sum(preds == labels.data)\n", "            \n", "            if phase == 'train':\n", "                train_epoch_loss = running_loss / dataset_sizes[phase]\n", "                train_epoch_acc = running_corrects / dataset_sizes[phase]\n", "            else:\n", "                valid_epoch_loss = running_loss / dataset_sizes[phase]\n", "                valid_epoch_acc = running_corrects / dataset_sizes[phase]\n", "                \n", "            if phase == 'valid' and valid_epoch_acc > best_acc:\n", "                best_acc = valid_epoch_acc\n", "                best_model_wts = model.state_dict()\n", "\n", "        print('Epoch [{}/{}] train loss: {:.4f} acc: {:.4f} ' \n", "              'valid loss: {:.4f} acc: {:.4f}'.format(\n", "                epoch, num_epochs - 1,\n", "                train_epoch_loss, train_epoch_acc, \n", "                valid_epoch_loss, valid_epoch_acc))\n", "            \n", "    print('Best val Acc: {:4f}'.format(best_acc))\n", "\n", "    model.load_state_dict(best_model_wts)\n", "    return model"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["resnet = models.resnet50(pretrained=True)\n", "# freeze all model parameters\n", "for param in resnet.parameters():\n", "    param.requires_grad = False\n", "\n", "# new final layer with 16 classes\n", "num_ftrs = resnet.fc.in_features\n", "resnet.fc = torch.nn.Linear(num_ftrs, 16)\n", "if use_gpu:\n", "    resnet = resnet.cuda()\n", "\n", "criterion = torch.nn.CrossEntropyLoss()\n", "optimizer = torch.optim.SGD(resnet.fc.parameters(), lr=0.001, momentum=0.9)\n", "exp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n", "\n", "dloaders = {'train':train_dl, 'valid':valid_dl}"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["start_time = time.time()\n", "model = train_model(dloaders, resnet, criterion, optimizer, exp_lr_scheduler, num_epochs=2)\n", "print('Training time: {:10f} minutes'.format((time.time()-start_time)/60))"], "outputs": []}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": ["def visualize_model(dataloders, model, num_images=16):\n", "    cnt = 0\n", "    fig = plt.figure(1, figsize=(16, 16))\n", "    grid = ImageGrid(fig, 111, nrows_ncols=(4, 4), axes_pad=0.05)\n", "    for i, (inputs, labels) in enumerate(dataloders['valid']):\n", "        if use_gpu:\n", "            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n", "        else:\n", "            inputs, labels = Variable(inputs), Variable(labels)\n", "\n", "        outputs = model(inputs)\n", "        _, preds = torch.max(outputs.data, 1)\n", "\n", "        for j in range(inputs.size()[0]):\n", "            ax = grid[cnt]\n", "            imshow(ax, inputs.cpu().data[j])\n", "            ax.text(10, 210, '{}/{}'.format(preds[j], labels.data[j]), \n", "                    color='k', backgroundcolor='w', alpha=0.8)\n", "            cnt += 1\n", "            if cnt == num_images:\n", "                return"], "outputs": []}, {"cell_type": "code", "metadata": {}, "execution_count": null, "source": ["visualize_model(dloaders, resnet)"], "outputs": []}, {"cell_type": "markdown", "metadata": {}, "source": ["This kernel was mainly to test using transfer learning in kernels using PyTorch. Training is slow in CPU but it works.   "]}, {"cell_type": "code", "metadata": {"collapsed": true}, "execution_count": null, "source": [], "outputs": []}]}