{"cells":[{"metadata":{"collapsed":true,"_uuid":"11db94b40434802ac10d5ed76e21d33b9ff6befe","_cell_guid":"f5af2eca-5b02-4d89-a36d-8ca330851ac3"},"cell_type":"markdown","source":"# Transfer learning in kernels with PyTorch\n\nFollowing the same strategy from Beluga's kernel [Use pretrained Keras models](https://www.kaggle.com/gaborfodor/use-pretrained-keras-models-lb-0-3), this kernel uses a dataset with PyTorch pretrained networks weights. \n\nTraining in the CPU is quite slow, but it is still feasible to use a pre-trained network, replace the final layer and train just this last layer. \n\nThanks Beluga for your great kernel. This one uses not only the concept but also a lot of the code. "},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport time\nimport numpy as np\nimport pandas as pd\nimport datetime as dt\nimport matplotlib.pyplot as plt\nfrom mpl_toolkits.axes_grid1 import ImageGrid\nfrom os import listdir, makedirs, getcwd, remove\nfrom os.path import isfile, join, abspath, exists, isdir, expanduser\nfrom PIL import Image\nimport torch\nfrom torch.optim import lr_scheduler\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision\nfrom torchvision import transforms, datasets, models","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This [dataset](https://www.kaggle.com/pvlima/pretrained-pytorch-models) has the PyTorch weights for some pre-trained networks.\n\nWe have to copy the pretrained models to the cache directory (~/.torch/models) where PyTorch is looking for them."},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/pretrained-pytorch-models/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cache_dir = expanduser(join('~', '.torch'))\nif not exists(cache_dir):\n    makedirs(cache_dir)\nmodels_dir = join(cache_dir, 'models')\nif not exists(models_dir):\n    makedirs(models_dir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!cp ../input/pretrained-pytorch-models/* /root/.cache/torch/checkpoints/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls /root/.cache/torch/checkpoints/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input/dog-breed-identification","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using just 16 most frequent breeds to keep the running time under the kernel limit"},{"metadata":{"trusted":true},"cell_type":"code","source":"INPUT_SIZE = 224\nNUM_CLASSES = 120\ndata_dir = '../input/dog-breed-identification/'\nlabels = pd.read_csv(join(data_dir, 'labels.csv'))\nsample_submission = pd.read_csv(join(data_dir, 'sample_submission.csv'))\nprint(len(listdir(join(data_dir, 'train'))), len(labels))\nprint(len(listdir(join(data_dir, 'test'))), len(sample_submission))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"selected_breed_list = list(labels.groupby('breed').count().sort_values(by='id', ascending=False).head(NUM_CLASSES).index)\nlabels = labels[labels['breed'].isin(selected_breed_list)]\nlabels['target'] = 1\n# labels['rank'] = labels.groupby('breed').rank()['id']\nlabels_pivot = labels.pivot('id', 'breed', 'target').reset_index().fillna(0)\n\ntrain = labels_pivot.sample(frac=0.8)\nvalid = labels_pivot[~labels_pivot['id'].isin(train['id'])]\nprint(train.shape, valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DogsDataset(Dataset):\n    def __init__(self, labels, root_dir, subset=False, transform=None):\n        self.labels = labels\n        self.root_dir = root_dir\n        self.transform = transform\n    \n    def __len__(self):\n        return len(self.labels)\n    \n    def __getitem__(self, idx):\n        img_name = '{}.jpg'.format(self.labels.iloc[idx, 0])\n        fullname = join(self.root_dir, img_name)\n        image = Image.open(fullname)\n        labels = self.labels.iloc[idx, 1:].values.astype('float')\n        labels = np.argmax(labels)\n        if self.transform:\n            image = self.transform(image)\n        return [image, labels]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"normalize = transforms.Normalize(\n   mean=[0.485, 0.456, 0.406],\n   std=[0.229, 0.224, 0.225]\n)\nds_trans = transforms.Compose([transforms.Scale(224),\n                               transforms.CenterCrop(224),\n                               transforms.ToTensor(),\n                               normalize])\ntrain_ds = DogsDataset(train, data_dir+'train/', transform=ds_trans)\nvalid_ds = DogsDataset(valid, data_dir+'train/', transform=ds_trans)\n\ntrain_dl = DataLoader(train_ds, batch_size=4, shuffle=True, num_workers=4)\nvalid_dl = DataLoader(valid_ds, batch_size=4, shuffle=True, num_workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def imshow(axis, inp):\n    \"\"\"Denormalize and show\"\"\"\n    inp = inp.numpy().transpose((1, 2, 0))\n    mean = np.array([0.485, 0.456, 0.406])\n    std = np.array([0.229, 0.224, 0.225])\n    inp = std * inp + mean\n    axis.imshow(inp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = next(iter(train_ds))\nprint(img.size(), label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"img, label = next(iter(train_dl))\nprint(img.size(), label)\nfig = plt.figure(1, figsize=(16, 4))\ngrid = ImageGrid(fig, 111, nrows_ncols=(1, 4), axes_pad=0.05)    \nfor i in range(img.size()[0]):\n    ax = grid[i]\n    imshow(ax, img[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ResNet50\n\n### Just try the model "},{"metadata":{"trusted":true},"cell_type":"code","source":"use_gpu = torch.cuda.is_available()\nresnet = models.resnet50(pretrained=True)\ninputs, labels = next(iter(train_dl))\nif use_gpu:\n    resnet = resnet.cuda()\n    inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())   \nelse:\n    inputs, labels = Variable(inputs), Variable(labels)\noutputs = resnet(inputs)\noutputs.size()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The model seems to work OK. Resnet outputs probabilities for the imagenet 1000 labels as expected. "},{"metadata":{},"cell_type":"markdown","source":"### Replace last layer and train\n\nWill replace the last layer with one that predicts the 16 classes. The network weights will be fixed expected for the last layer that is trained."},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(dataloders, model, criterion, optimizer, scheduler, num_epochs=25):\n    since = time.time()\n    use_gpu = torch.cuda.is_available()\n    best_model_wts = model.state_dict()\n    best_acc = 0.0\n    dataset_sizes = {'train': len(dataloders['train'].dataset), \n                     'valid': len(dataloders['valid'].dataset)}\n\n    for epoch in range(num_epochs):\n        for phase in ['train', 'valid']:\n            if phase == 'train':\n                scheduler.step()\n                model.train(True)\n            else:\n                model.train(False)\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            for inputs, labels in dataloders[phase]:\n                if use_gpu:\n                    inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n                else:\n                    inputs, labels = Variable(inputs), Variable(labels)\n\n                optimizer.zero_grad()\n\n                outputs = model(inputs) #[num,num * 60]\n                _, preds = torch.max(outputs.data, 1)\n                loss = criterion(outputs, labels)\n\n                if phase == 'train':\n                    loss.backward()\n                    optimizer.step()\n\n                running_loss += loss.data\n                running_corrects += torch.sum(preds == labels.data)\n            \n            if phase == 'train':\n                train_epoch_loss = running_loss / dataset_sizes[phase]\n                train_epoch_acc = float(running_corrects) / dataset_sizes[phase]\n            else:\n                valid_epoch_loss = running_loss / dataset_sizes[phase]\n                valid_epoch_acc = float(running_corrects) / dataset_sizes[phase]\n                \n            if phase == 'valid' and valid_epoch_acc > best_acc:\n                best_acc = valid_epoch_acc\n                best_model_wts = model.state_dict()\n\n        print('Epoch [{}/{}] train loss: {:.4f} acc: {:.4f} ' \n              'valid loss: {:.4f} acc: {:.4f}'.format(\n                epoch, num_epochs - 1,\n                train_epoch_loss, train_epoch_acc, \n                valid_epoch_loss, valid_epoch_acc))\n            \n    print('Best val Acc: {:4f}'.format(best_acc))\n\n    model.load_state_dict(best_model_wts)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resnet = models.resnet50(pretrained=True)\n# freeze all model parameters\nfor param in resnet.parameters():\n    param.requires_grad = False\n\n# new final layer with 16 classes\nnum_ftrs = resnet.fc.in_features\nresnet.fc = torch.nn.Linear(num_ftrs, 120)\nif use_gpu:\n    resnet = resnet.cuda()\n\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = torch.optim.Adam(resnet.fc.parameters(), lr=0.001)\nexp_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)\n\ndloaders = {'train':train_dl, 'valid':valid_dl}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nmodel = train_model(dloaders, resnet, criterion, optimizer, exp_lr_scheduler, num_epochs=20)\nprint('Training time: {:10f} minutes'.format((time.time()-start_time)/120))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PATH='/kaggle/working/model.pth'\ntorch.save(model, PATH)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> <a href=\"./model.pth\"> Download File </a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_model(dataloders, model, num_images=16):\n    cnt = 0\n    fig = plt.figure(1, figsize=(16, 16))\n    grid = ImageGrid(fig, 111, nrows_ncols=(4, 4), axes_pad=0.05)\n    for i, (inputs, labels) in enumerate(dataloders['valid']):\n        print('labels', labels)\n        if use_gpu:\n            inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n        else:\n            inputs, labels = Variable(inputs), Variable(labels)\n        outputs = model(inputs)\n        _, preds = torch.max(outputs.data, 1)\n        print('preds', preds)\n\n        for j in range(inputs.size()[0]):\n            ax = grid[cnt]\n            imshow(ax, inputs.cpu().data[j])\n            ax.text(10, 210, '{}/{}'.format(preds[j], labels.data[j]), \n                    color='k', backgroundcolor='w', alpha=0.8)\n            cnt += 1\n            if cnt == num_images:\n                return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"visualize_model(dloaders, model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This kernel was mainly to test using transfer learning in kernels using PyTorch. Training is slow in CPU but it works.   "},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}