{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nfrom torch.optim import lr_scheduler\nimport torch\nimport torchvision\nimport torchvision.transforms as transforms\nimport matplotlib.pyplot as plt\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom distutils.dir_util import copy_tree\nimport random\nimport copy\n\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nprint(device)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Process dataset, put dog images of the same breed in the same folder\n\nPATH='/kaggle/input/dog-breed-identification/'\nW_PATH='/kaggle/working/dog-breed-identification/'\nlabels = pd.read_csv(f'{PATH}labels.csv')\n\nos.mkdir(f'/kaggle/working/dog-breed-identification')\nos.mkdir(f'{W_PATH}valid')\nos.mkdir(f'{W_PATH}train')\nos.mkdir(f'{W_PATH}test')\nos.mkdir(f'{W_PATH}test/0')  # for allowing ImageFolder to work\n\nfor f in labels.breed.unique():\n    os.mkdir(f'{W_PATH}train/{f}')\n    os.mkdir(f'{W_PATH}valid/{f}')\n\nn = 0.1  # portion of validation set\n\n# Move the train and test data from input to working directory\ncopy_tree(f'{PATH}train', f'{W_PATH}train')\ncopy_tree(f'{PATH}test', f'{W_PATH}test/0')\n\n# Move the training data to subfolders by their labels\nfor p in labels.itertuples():\n    file_path = f'{W_PATH}train/{p.id}.jpg'\n    train_path = f'{W_PATH}train/{p.breed}/{p.id}.jpg'\n    os.rename(f'{file_path}', f'{train_path}')\n\n# Move a portion of validation data from training folders to validation folders\nfor f in os.listdir(f'{W_PATH}train'):\n    pics = os.listdir(f'{W_PATH}train/{f}')\n    num_pics = len(pics)\n    num_val_pics = round(n * num_pics)\n    val_pics = random.sample(pics, num_val_pics)\n    \n    for p in val_pics:\n        file_path = f'{W_PATH}train/{f}/{p}'\n        val_path = f'{W_PATH}valid/{f}/{p}'\n        os.rename(f'{file_path}', f'{val_path}')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_dogs_data(augmentation=0):\n    transform_train = transforms.Compose([\n        transforms.RandomResizedCrop(256),\n        transforms.RandomHorizontalFlip(),\n        transforms.RandomVerticalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    \n    transform_test = transforms.Compose([\n        transforms.Resize((256, 256)),\n        transforms.ToTensor(),\n        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n    ])\n    trainset = torchvision.datasets.ImageFolder(root=f'{W_PATH}train', transform=transform_train)\n    trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True, num_workers=2)\n    \n    validset = torchvision.datasets.ImageFolder(root=f'{W_PATH}valid', transform=transform_test)\n    validloader = torch.utils.data.DataLoader(validset, batch_size=128, shuffle=False, num_workers=2)\n    \n    testset = torchvision.datasets.ImageFolder(root=f'{W_PATH}test', transform=transform_test)\n    testloader = torch.utils.data.DataLoader(testset, batch_size=1, shuffle=False, num_workers=2)\n    \n    return {'train': trainloader, 'valid': validloader, 'test': testloader, 'classes': trainloader.dataset.classes}\n\ndata = get_dogs_data()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Visualization\n\ndef smooth(x, size):\n  return np.convolve(x, np.ones(size)/size, mode='valid')\n\ndataiter = iter(data['train'])\nimages, labels = dataiter.next()\nimages = images[:8]\nprint(images.size())\n\ndef imshow(img):\n    npimg = img.numpy()\n    plt.imshow(np.transpose(npimg, (1, 2, 0)))\n    plt.show()\n\n# show images\nimshow(torchvision.utils.make_grid(images))\n# print labels\nprint(\"Labels:\" + ', '.join('%9s' % data['classes'][labels[j]] for j in range(8)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Method for training the neural network model\n### AUTHOR Yixiao Li\n\n\ndef train_model(model, data, optimizer, scheduler, loss_func, num_epochs=5):\n    \"\"\"\n    Trains the model on train set and evaluates on validation set\n\n    Arguments:\n        model is the model to be trained and evaluated\n        data is a dictionary that contains mappings for the train and validation set\n        optimizer is used to optimize the model during training\n        loss_func is the loss function\n        num_epochs is a positive integer\n\n    Returns:\n        the loss on train set over epochs\n        the loss on validation set over epochs\n        the highest validation accuracy\n        the model with the highest validation accuracy\n    \"\"\"\n    train_losses = []\n    valid_losses = []\n    best_valid_accuracy = 0.0\n    best_model_state = copy.deepcopy(model.state_dict())\n\n    for epoch in range(num_epochs):\n        print(epoch + 1, \"/\", num_epochs, \"epochs\")  # keep track of progress\n\n        # there is a training phase and validation phase for each epoch\n        for phase in ['train', 'valid']:                \n            if phase == 'train':\n                model = model.train()  # set model to training mode\n            else:\n                model = model.eval()   # set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # iterate over the dataset in batches using dataloader\n            for inputs, labels in data[phase]:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # zero the parameter gradients\n                optimizer.zero_grad()\n\n                # forward; track history only if training\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, predicts = torch.max(outputs, 1)\n                    loss = loss_func(outputs, labels)\n\n                    # backward + optimize only if training\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # add iteration statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(predicts == labels.data)\n\n            if phase == 'train':\n                scheduler.step()\n            \n            # epoch statistics\n            num_data = len(data[phase].dataset)\n            epoch_loss = running_loss / num_data\n            epoch_accuracy = running_corrects.double() / num_data\n\n            # record epoch loss\n            if phase == 'train':\n                print('train loss: ', epoch_loss)\n                train_losses.append(epoch_loss)\n            else:\n                print('validation loss: ', epoch_loss)\n                valid_losses.append(epoch_loss)\n\n            # deep copy the model only if validating and model has highest validation accuracy\n            if phase == 'valid' and epoch_accuracy > best_valid_accuracy:\n                best_valid_accuracy = epoch_accuracy\n                best_model_state = copy.deepcopy(model.state_dict())\n\n    # load best model\n    model.load_state_dict(best_model_state)\n\n    return train_losses, valid_losses, best_valid_accuracy, model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Configurations\n\n# number of classes\nnum_classes = 120\n\n# initialize for transfer learning\nfine_tune = False\n\n### model\n#model = torch.hub.load('pytorch/vision:v0.6.0', 'inception_v3', pretrained=True, aux_logits=False)\n#model = model = torch.hub.load('pytorch/vision:v0.6.0', 'resnext50_32x4d', pretrained=True)\nmodel = torch.hub.load('facebookresearch/WSL-Images', 'resnext101_32x8d_wsl')\n#model = torch.hub.load('facebookresearch/semi-supervised-ImageNet1K-models', 'resnext101_32x8d_swsl')\n\nif not fine_tune:\n    for param in model.parameters():\n        param.requires_grad = False\n    model.fc.requires_grad = True\nmodel.fc = nn.Sequential(\n            nn.Linear(2048, 512),\n            nn.ReLU(),\n            nn.Linear(512, num_classes)\n        )\nmodel = model.to(device)\n\n# loss function\nloss_func = torch.nn.functional.cross_entropy\n\n# learning rate\nlr = 0.003\n\n# weight decay\nweight_decay = 0.0005\n\n# optimizer\noptimizer = None\nif not fine_tune:\n    optimizer = torch.optim.Adam(model.fc.parameters(), lr=lr, weight_decay=weight_decay)\nelse:\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay)\n\n# step_size\nstep_size = 7\n\n# gamma\ngamma = 0.3\n\n# scheduler\nscheduler = lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Training\n\n# number of epochs\nnum_epochs = 32\n\n# output important configurations\nprint('learning rate ', lr, ' weight decay ', weight_decay)\n\n# train model\ntrain_loss, valid_loss, highest_accuracy, model = train_model(model, data, optimizer, scheduler,\n                                                                 loss_func, num_epochs)\n\n# plot\nfig, ax = plt.subplots()\nax.plot(train_loss, label=\"train\", color=\"blue\")\nax.plot(valid_loss, label=\"validation\", color=\"red\")\nax.set_title(\"Cross Entropy Loss vs. Epochs Graph\")\nax.set_xlabel(\"Epochs\")\nax.set_ylabel(\"Loss\")\nax.legend()\n\nplt.show()\n\nprint(\"Highest accuracy on validation set is \", highest_accuracy.item())\nprint()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Save the best model state\n\nmodel_state_path = '../working/model/'\nif not os.path.exists(model_state_path):\n    os.makedirs(model_state_path)\n\nstate = model.state_dict()\ntorch.save(state, model_state_path + 'optimal_state')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nFileLink(r'model/optimal_state')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Make predictions on test data samples \n### (From https://www.kaggle.com/pjreddie/transfer-learning-to-birds-with-resnet18/notebook)\n\ndef predict(net, dataloader, ofname):\n    out = open(ofname, 'w')\n    out.write(\"path,class\\n\")\n    net.to(device)\n    net.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for i, (images, labels) in enumerate(dataloader, 0):\n            if i%100 == 0:\n                print(i)\n            images, labels = images.to(device), labels.to(device)\n            outputs = net(images)\n            _, predicted = torch.max(outputs.data, 1)\n            fname, _ = dataloader.dataset.samples[i]\n            out.write(\"test/{},{}\\n\".format(fname.split('/')[-1], data['classes'][predicted.item()]))\n    out.close()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict(model, data['test'], f'{W_PATH}preds.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}