{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### https://www.kaggle.com/c/cifar-10","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2022-01-30T23:51:21.508399Z","iopub.execute_input":"2022-01-30T23:51:21.508739Z","iopub.status.idle":"2022-01-30T23:51:21.538013Z","shell.execute_reply.started":"2022-01-30T23:51:21.508652Z","shell.execute_reply":"2022-01-30T23:51:21.53735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:51:21.539608Z","iopub.execute_input":"2022-01-30T23:51:21.540064Z","iopub.status.idle":"2022-01-30T23:51:32.631669Z","shell.execute_reply.started":"2022-01-30T23:51:21.540029Z","shell.execute_reply":"2022-01-30T23:51:32.630877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:51:32.634659Z","iopub.execute_input":"2022-01-30T23:51:32.634971Z","iopub.status.idle":"2022-01-30T23:51:35.217899Z","shell.execute_reply.started":"2022-01-30T23:51:32.634929Z","shell.execute_reply":"2022-01-30T23:51:35.217121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:51:35.21921Z","iopub.execute_input":"2022-01-30T23:51:35.219468Z","iopub.status.idle":"2022-01-30T23:51:43.286283Z","shell.execute_reply.started":"2022-01-30T23:51:35.219435Z","shell.execute_reply":"2022-01-30T23:51:43.285448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:51:43.290378Z","iopub.execute_input":"2022-01-30T23:51:43.290612Z","iopub.status.idle":"2022-01-30T23:52:26.349395Z","shell.execute_reply.started":"2022-01-30T23:51:43.290582Z","shell.execute_reply":"2022-01-30T23:52:26.347993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2022-01-30T23:52:26.351468Z","iopub.execute_input":"2022-01-30T23:52:26.351774Z","iopub.status.idle":"2022-01-31T00:06:23.327942Z","shell.execute_reply.started":"2022-01-30T23:52:26.351715Z","shell.execute_reply":"2022-01-31T00:06:23.326968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.330033Z","iopub.execute_input":"2022-01-31T00:06:23.330366Z","iopub.status.idle":"2022-01-31T00:06:23.334747Z","shell.execute_reply.started":"2022-01-31T00:06:23.330324Z","shell.execute_reply":"2022-01-31T00:06:23.334089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.335944Z","iopub.execute_input":"2022-01-31T00:06:23.336349Z","iopub.status.idle":"2022-01-31T00:06:23.413911Z","shell.execute_reply.started":"2022-01-31T00:06:23.336311Z","shell.execute_reply":"2022-01-31T00:06:23.413171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.415256Z","iopub.execute_input":"2022-01-31T00:06:23.415746Z","iopub.status.idle":"2022-01-31T00:06:23.425282Z","shell.execute_reply.started":"2022-01-31T00:06:23.415687Z","shell.execute_reply":"2022-01-31T00:06:23.424596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.426879Z","iopub.execute_input":"2022-01-31T00:06:23.427412Z","iopub.status.idle":"2022-01-31T00:06:23.435393Z","shell.execute_reply.started":"2022-01-31T00:06:23.427373Z","shell.execute_reply":"2022-01-31T00:06:23.434665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.437915Z","iopub.execute_input":"2022-01-31T00:06:23.438902Z","iopub.status.idle":"2022-01-31T00:06:23.447679Z","shell.execute_reply.started":"2022-01-31T00:06:23.438872Z","shell.execute_reply":"2022-01-31T00:06:23.446901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 256\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:06:23.449231Z","iopub.execute_input":"2022-01-31T00:06:23.449857Z","iopub.status.idle":"2022-01-31T00:07:21.149124Z","shell.execute_reply.started":"2022-01-31T00:06:23.449815Z","shell.execute_reply":"2022-01-31T00:07:21.148365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.RandomResizedCrop((32, 32), scale=(0.5, 1), ratio=(0.5, 2)),\n    torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.2),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])# HorizontalFlip, RandomSizeCrop, \"small\" Color Jitter\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:21.15054Z","iopub.execute_input":"2022-01-31T00:07:21.150809Z","iopub.status.idle":"2022-01-31T00:07:21.15884Z","shell.execute_reply.started":"2022-01-31T00:07:21.150773Z","shell.execute_reply":"2022-01-31T00:07:21.157783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntraining_size = len(train_ds)\ntrain_valid_size = len(train_valid_ds)\ntest_valid_size = len(valid_ds)\ntest_size = len(test_ds)\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:21.163025Z","iopub.execute_input":"2022-01-31T00:07:21.163299Z","iopub.status.idle":"2022-01-31T00:07:24.326741Z","shell.execute_reply.started":"2022-01-31T00:07:21.163263Z","shell.execute_reply":"2022-01-31T00:07:24.326043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit to Kaggle your submission file and get your accuracy on the test set","metadata":{}},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"WANDB_KEY\")\n\nimport wandb\nwandb.login(key=secret_value_0)\nwandb.init(project=\"pretraining-resnet18\")","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:24.328045Z","iopub.execute_input":"2022-01-31T00:07:24.328291Z","iopub.status.idle":"2022-01-31T00:07:32.772769Z","shell.execute_reply.started":"2022-01-31T00:07:24.328258Z","shell.execute_reply":"2022-01-31T00:07:32.77197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_net = torchvision.models.resnet18(pretrained=True)\n\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\n\npretrained_net","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:32.774233Z","iopub.execute_input":"2022-01-31T00:07:32.77465Z","iopub.status.idle":"2022-01-31T00:07:34.931644Z","shell.execute_reply.started":"2022-01-31T00:07:32.774607Z","shell.execute_reply":"2022-01-31T00:07:34.930842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.AdamW(pretrained_net.parameters(), lr=1e-3, weight_decay=0.0001)\n\nnum_epochs = 100\n\nfor epoch in range(num_epochs):\n    print(\"Epoch: \" + str(epoch+1) + \"/\" + str(num_epochs))\n    \n    #TRAINING LOOP\n    training_loss = 0\n    correct_classified = 0\n    pretrained_net.train()\n    for train_features, train_labels in train_iter:\n        train_features = train_features.to(device)\n        train_labels = train_labels.to(device)\n        \n        optimizer.zero_grad() # please don't forget!\n        \n        out = pretrained_net(train_features)\n        l = criterion(out, train_labels)\n        \n        l.backward() # compute gradient\n        optimizer.step() # do a step in the gradient direction\n        \n        training_loss += l.item()\n        #Sum correct classified...\n        _, predicted = torch.max(out.data, 1)\n        correct_classified += int(predicted.eq(train_labels).sum().item())\n        \n    train_acc = correct_classified / training_size\n    \n    #VALIDATION LOOP - With parts from Train-Set (With Data-Augmentation)\n    validation_train_loss = 0\n    correct_classified = 0\n    pretrained_net.eval()\n    with torch.no_grad():\n        for val_features, val_labels in train_valid_iter:\n            val_features = val_features.to(device)\n            val_labels = val_labels.to(device)\n            \n            out = pretrained_net(val_features)\n            l = criterion(out, val_labels)\n            \n            validation_train_loss += l.item()\n            #Sum correct classified...\n            _, predicted = torch.max(out.data, 1)\n            correct_classified += int(predicted.eq(train_labels).sum().item())\n     \n    val_train_acc = correct_classified / train_valid_size \n    \n    #VALIDATION LOOP - With parts from Test-Set (Without Data-Augmentation)\n    validation_test_loss = 0\n    correct_classified = 0\n    with torch.no_grad():\n        for val_features, val_labels in valid_iter:\n            val_features = val_features.to(device)\n            val_labels = val_labels.to(device)\n            \n            out = pretrained_net(val_features)\n            l = criterion(out, val_labels)\n            \n            validation_test_loss += l.item()\n            #Sum correct classified...\n            _, predicted = torch.max(out.data, 1)\n            correct_classified += int(predicted.eq(train_labels).sum().item())\n     \n    val_test_acc = correct_classified / test_valid_size \n            \n    wandb.log({'training_loss': training_loss, 'validation_test_loss': validation_test_loss, 'validation_train_loss': validation_train_loss,'validation_train_accuracy': val_train_acc, 'validation_test_accuracy': val_test_acc, 'training_accuracy': train_acc})\n\n    \n# SAVE THE MODEL\ntorch.save(pretrained_net.state_dict(), 'my_pretrained_net_' + wandb.run.name + '_' + wandb.run.id +'.pt')\nprint(\"finished...\")\n\n","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:38.08067Z","iopub.execute_input":"2022-01-31T00:07:38.081323Z","iopub.status.idle":"2022-01-31T03:36:59.494595Z","shell.execute_reply.started":"2022-01-31T00:07:38.081284Z","shell.execute_reply":"2022-01-31T03:36:59.493788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#freeze layers\n#for param in pretrained_net.parameters():\n#    param.requires_grad = False\n\n#swap last layer\npretrained_net.fc = nn.Linear(pretrained_net.fc.in_features, 10)\nnn.init.xavier_normal_(pretrained_net.fc.weight)\nnn.init.constant_(pretrained_net.fc.bias, 0);\n\npretrained_net = pretrained_net.to(device)\n\ndevice\n","metadata":{"execution":{"iopub.status.busy":"2022-01-31T00:07:34.935374Z","iopub.execute_input":"2022-01-31T00:07:34.935669Z","iopub.status.idle":"2022-01-31T00:07:38.079225Z","shell.execute_reply.started":"2022-01-31T00:07:34.93563Z","shell.execute_reply":"2022-01-31T00:07:38.078519Z"},"trusted":true},"execution_count":null,"outputs":[]}]}