{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### Kaggle notebook: https://www.kaggle.com/code/paulabal/cifar-10-object-recognition\n\n#### Wandb Logs: https://wandb.ai/lefugdot/cifar10-resnet18/workspace?workspace=user-lefugdot","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T13:21:11.846094Z","iopub.execute_input":"2024-01-22T13:21:11.846473Z","iopub.status.idle":"2024-01-22T13:21:11.858295Z","shell.execute_reply.started":"2024-01-22T13:21:11.846445Z","shell.execute_reply":"2024-01-22T13:21:11.857010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\n\nfrom kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret = user_secrets.get_secret(\"wandb-key\")\n\nwandb.login(key=secret)\nwandb.init(project=\"cifar10-resnet18\")","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:19:12.108376Z","iopub.execute_input":"2024-01-22T13:19:12.108746Z","iopub.status.idle":"2024-01-22T13:19:49.019565Z","shell.execute_reply.started":"2024-01-22T13:19:12.108710Z","shell.execute_reply":"2024-01-22T13:19:49.018664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset\nfrom torchvision.datasets import CIFAR10\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:21:14.174503Z","iopub.execute_input":"2024-01-22T13:21:14.174878Z","iopub.status.idle":"2024-01-22T13:21:14.186396Z","shell.execute_reply.started":"2024-01-22T13:21:14.174844Z","shell.execute_reply":"2024-01-22T13:21:14.185347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-21T22:01:39.973008Z","iopub.execute_input":"2024-01-21T22:01:39.973925Z","iopub.status.idle":"2024-01-21T22:01:55.351009Z","shell.execute_reply.started":"2024-01-21T22:01:39.973889Z","shell.execute_reply":"2024-01-21T22:01:55.349980Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-21T22:03:13.008416Z","iopub.execute_input":"2024-01-21T22:03:13.009096Z","iopub.status.idle":"2024-01-21T22:03:15.159945Z","shell.execute_reply.started":"2024-01-21T22:03:13.009060Z","shell.execute_reply":"2024-01-21T22:03:15.158958Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-21T22:03:18.466381Z","iopub.execute_input":"2024-01-21T22:03:18.467257Z","iopub.status.idle":"2024-01-21T22:21:56.895405Z","shell.execute_reply.started":"2024-01-21T22:03:18.467221Z","shell.execute_reply":"2024-01-21T22:21:56.894307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:21:18.629548Z","iopub.execute_input":"2024-01-22T13:21:18.629896Z","iopub.status.idle":"2024-01-22T13:21:18.636190Z","shell.execute_reply.started":"2024-01-22T13:21:18.629869Z","shell.execute_reply":"2024-01-22T13:21:18.634907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:21:20.713346Z","iopub.execute_input":"2024-01-22T13:21:20.713751Z","iopub.status.idle":"2024-01-22T13:21:22.726595Z","shell.execute_reply.started":"2024-01-22T13:21:20.713712Z","shell.execute_reply":"2024-01-22T13:21:22.725353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T01:22:59.075041Z","iopub.execute_input":"2024-01-22T01:22:59.075931Z","iopub.status.idle":"2024-01-22T01:22:59.086623Z","shell.execute_reply.started":"2024-01-22T01:22:59.075897Z","shell.execute_reply":"2024-01-22T01:22:59.085786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:21:26.325744Z","iopub.execute_input":"2024-01-22T13:21:26.326116Z","iopub.status.idle":"2024-01-22T13:21:26.331789Z","shell.execute_reply.started":"2024-01-22T13:21:26.326088Z","shell.execute_reply":"2024-01-22T13:21:26.330750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    transforms.RandomRotation(15),\n    transforms.RandomCrop(30, padding=4),\n    transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1),\n    transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:22:46.819438Z","iopub.execute_input":"2024-01-22T13:22:46.820393Z","iopub.status.idle":"2024-01-22T13:22:46.829265Z","shell.execute_reply.started":"2024-01-22T13:22:46.820357Z","shell.execute_reply":"2024-01-22T13:22:46.827992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:22:48.980382Z","iopub.execute_input":"2024-01-22T13:22:48.980732Z","iopub.status.idle":"2024-01-22T13:22:50.950474Z","shell.execute_reply.started":"2024-01-22T13:22:48.980704Z","shell.execute_reply":"2024-01-22T13:22:50.949400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"markdown","source":"#### Model fine-tuning, training & validation loops ","metadata":{}},{"cell_type":"code","source":"device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n\nmodel = models.resnet18(weights='ResNet18_Weights.DEFAULT')\n\nnum_ftrs = model.fc.in_features\nmodel.fc = nn.Sequential(\n    nn.Linear(num_ftrs, 512),\n    nn.BatchNorm1d(512),\n    nn.ReLU(),\n    nn.Linear(512, 10)\n)\nmodel = model.to(device)\n\nlr = 0.001\nmin_lr = 0.0001\nepochs = 20\n\ncriterion = torch.nn.CrossEntropyLoss()\noptimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=0.001)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, epochs, eta_min=min_lr)\n\nfor epoch in range(epochs):\n    model.train()\n    running_loss = 0.0\n    \n    correct = 0\n    total = 0\n    for i, data in enumerate(train_iter):\n        inputs, labels = data[0].to(device), data[1].to(device)\n\n        optimizer.zero_grad()\n\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n\n        running_loss += loss.item()\n        \n        _, predicted = outputs.max(1)\n        total += labels.size(0)\n        correct += predicted.eq(labels).sum().item()\n\n    wandb.log({\"train_loss\": running_loss / len(train_iter), \"train_accuracy\": correct / total})\n\n    model.eval()\n    valid_loss = 0.0\n    \n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for i, data in enumerate(valid_iter):\n            inputs, labels = data[0].to(device), data[1].to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            valid_loss += loss.item()\n            \n            _, predicted = outputs.max(1)\n            total += labels.size(0)\n            correct += predicted.eq(labels).sum().item()\n\n    scheduler.step()\n    wandb.log({\"valid_loss\": valid_loss / len(valid_iter), \"valid_accuracy\": correct / total})","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:23:16.251629Z","iopub.execute_input":"2024-01-22T13:23:16.252008Z","iopub.status.idle":"2024-01-22T13:45:33.721143Z","shell.execute_reply.started":"2024-01-22T13:23:16.251976Z","shell.execute_reply":"2024-01-22T13:45:33.717505Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Save the test results into the submission files","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ndef get_image_id(image_path):\n    start_index = len(os.path.join(data_dir, 'train_valid_test', 'test', 'unknown')) + 1\n    end_index = -4\n    return image_path[start_index:end_index]\n\nmodel.eval()\ntest_pred = torch.LongTensor()\nimage_ids = []\n\nall_image_paths = [img_path for img_path, _ in test_iter.dataset.imgs]\n\nclasses = ('airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\nimage_ids = []\npredicted_names_list = []\n\nwith torch.no_grad():\n    for i, data in enumerate(test_iter):\n        inputs, labels = data[0].to(device), data[1].to(device)\n        output = model(inputs)\n        _, predicted = torch.max(output.data, 1)\n        predicted = predicted.cpu()\n\n        predicted_names = [classes[p] for p in predicted]\n        predicted_names_list.extend(predicted_names)\n\n        batch_start = i * batch_size\n        batch_end = batch_start + batch_size\n        batch_image_paths = all_image_paths[batch_start:batch_end]\n\n        for image_path in batch_image_paths:\n            image_ids.append(get_image_id(image_path))\n\nout_df = pd.DataFrame(np.c_[image_ids, predicted_names_list], columns=['id', 'label'])\n\nout_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:11:43.493133Z","iopub.execute_input":"2024-01-22T14:11:43.494080Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]}]}