{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-30T21:23:09.154261Z","iopub.execute_input":"2022-01-30T21:23:09.154690Z","iopub.status.idle":"2022-01-30T21:23:09.187598Z","shell.execute_reply.started":"2022-01-30T21:23:09.154576Z","shell.execute_reply":"2022-01-30T21:23:09.186925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import everything needed!","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:23:09.188759Z","iopub.execute_input":"2022-01-30T21:23:09.191079Z","iopub.status.idle":"2022-01-30T21:23:11.621452Z","shell.execute_reply.started":"2022-01-30T21:23:09.191041Z","shell.execute_reply":"2022-01-30T21:23:11.620423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:23:11.622765Z","iopub.execute_input":"2022-01-30T21:23:11.623030Z","iopub.status.idle":"2022-01-30T21:23:23.845621Z","shell.execute_reply.started":"2022-01-30T21:23:11.622994Z","shell.execute_reply":"2022-01-30T21:23:23.844585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:23:23.847532Z","iopub.execute_input":"2022-01-30T21:23:23.847855Z","iopub.status.idle":"2022-01-30T21:24:09.031714Z","shell.execute_reply.started":"2022-01-30T21:23:23.847809Z","shell.execute_reply":"2022-01-30T21:24:09.030820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z # Around 15 minutes to uncompress.","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:24:09.036175Z","iopub.execute_input":"2022-01-30T21:24:09.036420Z","iopub.status.idle":"2022-01-30T21:38:08.863003Z","shell.execute_reply.started":"2022-01-30T21:24:09.036388Z","shell.execute_reply":"2022-01-30T21:38:08.862104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.864649Z","iopub.execute_input":"2022-01-30T21:38:08.865063Z","iopub.status.idle":"2022-01-30T21:38:08.871886Z","shell.execute_reply.started":"2022-01-30T21:38:08.865018Z","shell.execute_reply":"2022-01-30T21:38:08.871183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.874571Z","iopub.execute_input":"2022-01-30T21:38:08.874788Z","iopub.status.idle":"2022-01-30T21:38:08.950655Z","shell.execute_reply.started":"2022-01-30T21:38:08.874762Z","shell.execute_reply":"2022-01-30T21:38:08.949993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.951986Z","iopub.execute_input":"2022-01-30T21:38:08.952228Z","iopub.status.idle":"2022-01-30T21:38:08.962604Z","shell.execute_reply.started":"2022-01-30T21:38:08.952195Z","shell.execute_reply":"2022-01-30T21:38:08.961816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.963781Z","iopub.execute_input":"2022-01-30T21:38:08.964044Z","iopub.status.idle":"2022-01-30T21:38:08.970525Z","shell.execute_reply.started":"2022-01-30T21:38:08.964011Z","shell.execute_reply":"2022-01-30T21:38:08.969590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.971606Z","iopub.execute_input":"2022-01-30T21:38:08.972001Z","iopub.status.idle":"2022-01-30T21:38:08.981441Z","shell.execute_reply.started":"2022-01-30T21:38:08.971965Z","shell.execute_reply":"2022-01-30T21:38:08.980731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:38:08.983337Z","iopub.execute_input":"2022-01-30T21:38:08.983766Z","iopub.status.idle":"2022-01-30T21:39:08.696151Z","shell.execute_reply.started":"2022-01-30T21:38:08.983710Z","shell.execute_reply":"2022-01-30T21:39:08.695331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Wandb Setup","metadata":{}},{"cell_type":"code","source":"import wandb\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"wandbkey\") # load wandbkey from Kaggle's secrets\n\nwandb.login(key=secret_value_0)\nwandb.init(project='Cifar 10', save_code=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:08.697472Z","iopub.execute_input":"2022-01-30T21:39:08.697854Z","iopub.status.idle":"2022-01-30T21:39:17.696668Z","shell.execute_reply.started":"2022-01-30T21:39:08.697816Z","shell.execute_reply":"2022-01-30T21:39:17.695961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Fine-Tuning","metadata":{}},{"cell_type":"code","source":"# Define the data augmentation technique \ntransform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomResizedCrop(32), # RandomSizeCrop, randomly crops and resize to given value\n    torchvision.transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5,hue=0.5), # Color Jitter\n    torchvision.transforms.RandomHorizontalFlip(), # Horizontal Flip\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:17.698174Z","iopub.execute_input":"2022-01-30T21:39:17.699275Z","iopub.status.idle":"2022-01-30T21:39:18.302734Z","shell.execute_reply.started":"2022-01-30T21:39:17.699234Z","shell.execute_reply":"2022-01-30T21:39:18.301822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:18.306125Z","iopub.execute_input":"2022-01-30T21:39:18.306386Z","iopub.status.idle":"2022-01-30T21:39:22.035816Z","shell.execute_reply.started":"2022-01-30T21:39:18.306353Z","shell.execute_reply":"2022-01-30T21:39:22.035146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fine-tune a pre-trained ResNet-18 on CIFAR-10\n# We swap the head with a Linear layer of the correct output shape that we initialize\npretrained_net = torchvision.models.resnet18(pretrained=True)\npretrained_net.fc = nn.Linear(pretrained_net.fc.in_features, 10)\nnn.init.xavier_normal_(pretrained_net.fc.weight)\nnn.init.constant_(pretrained_net.fc.bias, 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:22.036969Z","iopub.execute_input":"2022-01-30T21:39:22.038998Z","iopub.status.idle":"2022-01-30T21:39:23.372682Z","shell.execute_reply.started":"2022-01-30T21:39:22.038961Z","shell.execute_reply":"2022-01-30T21:39:23.371869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If we have a GPU, we use it. DataParallel allows to use multiple GPU in parallel if we have them\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\npretrained_net = pretrained_net.to(device)\nif device == 'cuda':\n    pretrained_net = torch.nn.DataParallel(pretrained_net) # if multiple GPUs use them\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(pretrained_net.parameters(), lr=1e-4, weight_decay=0.0001)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=3, verbose=True, min_lr=1e-5, factor=0.5)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:23.374143Z","iopub.execute_input":"2022-01-30T21:39:23.374403Z","iopub.status.idle":"2022-01-30T21:39:27.242519Z","shell.execute_reply.started":"2022-01-30T21:39:23.374368Z","shell.execute_reply":"2022-01-30T21:39:27.241847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import trange, tqdm\n\nepochs = 10\nfor epoch in trange(epochs):\n    accurate = 0\n    total = 0\n    losses = 0\n    for X, y in tqdm(train_iter): # Training loop\n        # Send to GPU\n        X = X.to(device)\n        y = y.to(device)\n        \n        y_pred = pretrained_net(X)\n        loss = criterion(y_pred, y)\n        score, predicted = torch.max(y_pred, 1)\n        accurate += (y == predicted).sum().float()\n        losses += loss.item()\n        total += len(y)\n\n        # zero the gradients before running\n        # the backward pass.\n        optimizer.zero_grad()\n\n        # Backward pass to compute the gradient\n        # of loss w.r.t our learnable params. \n        loss.backward()\n\n        # Update params\n        optimizer.step()\n    \n    wandb.log({\n            'training-loss': losses / len(train_iter),\n            'training-accuracy': accurate / total\n    })\n    with torch.no_grad(): # Validation loop\n        pretrained_net.eval()\n        accurate = 0\n        total = 0 \n        losses = 0\n        for X, y in tqdm(valid_iter):\n            # Send to GPU\n            X = X.to(device)\n            y = y.to(device)\n                \n            y_pred = pretrained_net(X)\n            loss = criterion(y_pred, y)\n            score, predicted = torch.max(y_pred, 1)\n            accurate += (y == predicted).sum().float()\n            losses += loss.item()\n            total += len(y)\n        wandb.log({\n            'validation-loss': losses / len(valid_iter),\n            'validation-accuracy': accurate / total\n        })","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:39:27.246843Z","iopub.execute_input":"2022-01-30T21:39:27.248794Z","iopub.status.idle":"2022-01-30T21:51:01.215229Z","shell.execute_reply.started":"2022-01-30T21:39:27.248755Z","shell.execute_reply":"2022-01-30T21:51:01.214435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"torch.save(pretrained_net.state_dict(), 'model.pt') # Save de model","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:51:01.216412Z","iopub.execute_input":"2022-01-30T21:51:01.216847Z","iopub.status.idle":"2022-01-30T21:51:01.863054Z","shell.execute_reply.started":"2022-01-30T21:51:01.216805Z","shell.execute_reply":"2022-01-30T21:51:01.862328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction Submission","metadata":{}},{"cell_type":"code","source":"# Based on \"Generating Predictions\" from \n# https://www.kaggle.com/francescolorenzo/96-fine-tuning-resnet34-with-pytorch submitted 7 months ago.\npreds = []\n\npretrained_net.eval()\nwith torch.no_grad():\n    for X, _ in tqdm(test_iter):\n        X = X.to(device) # Send to GPU\n        preds.extend(pretrained_net(X).argmax(dim=1).type(torch.int32).cpu().numpy()) # Collect predictions\nids = list(range(1, len(test_ds)+1)) # Get IDs from the dataset\nids.sort(key=lambda x: str(x)) # Sort the IDs\ndf = pd.DataFrame({'id': ids, 'label': preds}) # Dataframe creation\ndf['label'] = df['label'].apply(lambda x: train_ds.classes[x]) # Set the class predicted in the prediction column,i.e.cat  \ndf.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:51:01.866097Z","iopub.execute_input":"2022-01-30T21:51:01.866328Z","iopub.status.idle":"2022-01-30T21:53:58.564920Z","shell.execute_reply.started":"2022-01-30T21:51:01.866283Z","shell.execute_reply":"2022-01-30T21:53:58.564182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load saved model for future stuff\npretrained_net.load_state_dict(torch.load('model.pt'))","metadata":{"execution":{"iopub.status.busy":"2022-01-30T21:53:58.566244Z","iopub.execute_input":"2022-01-30T21:53:58.566745Z","iopub.status.idle":"2022-01-30T21:53:59.161293Z","shell.execute_reply.started":"2022-01-30T21:53:58.566705Z","shell.execute_reply":"2022-01-30T21:53:59.160448Z"},"trusted":true},"execution_count":null,"outputs":[]}]}