{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T09:46:37.110381Z","iopub.execute_input":"2024-01-22T09:46:37.110759Z","iopub.status.idle":"2024-01-22T09:46:37.530809Z","shell.execute_reply.started":"2024-01-22T09:46:37.110728Z","shell.execute_reply":"2024-01-22T09:46:37.529843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nimport torchvision.models as models\nimport wandb\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:03:07.972412Z","iopub.execute_input":"2024-01-22T10:03:07.972801Z","iopub.status.idle":"2024-01-22T10:03:08.761432Z","shell.execute_reply.started":"2024-01-22T10:03:07.972766Z","shell.execute_reply":"2024-01-22T10:03:08.760699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:09:25.228725Z","iopub.execute_input":"2024-01-22T09:09:25.229093Z","iopub.status.idle":"2024-01-22T09:28:04.931029Z","shell.execute_reply.started":"2024-01-22T09:09:25.22906Z","shell.execute_reply":"2024-01-22T09:28:04.929767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:46:00.427622Z","iopub.execute_input":"2024-01-22T09:46:00.428527Z","iopub.status.idle":"2024-01-22T09:46:00.432891Z","shell.execute_reply.started":"2024-01-22T09:46:00.428482Z","shell.execute_reply":"2024-01-22T09:46:00.431919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:46:45.307477Z","iopub.execute_input":"2024-01-22T09:46:45.308221Z","iopub.status.idle":"2024-01-22T09:46:45.387435Z","shell.execute_reply.started":"2024-01-22T09:46:45.30818Z","shell.execute_reply":"2024-01-22T09:46:45.386432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:46:51.405107Z","iopub.execute_input":"2024-01-22T09:46:51.405767Z","iopub.status.idle":"2024-01-22T09:46:51.41672Z","shell.execute_reply.started":"2024-01-22T09:46:51.405729Z","shell.execute_reply":"2024-01-22T09:46:51.415709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:46:58.465086Z","iopub.execute_input":"2024-01-22T09:46:58.465764Z","iopub.status.idle":"2024-01-22T09:47:58.97227Z","shell.execute_reply.started":"2024-01-22T09:46:58.465729Z","shell.execute_reply":"2024-01-22T09:47:58.971362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomCrop(32, padding=4),\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:50:02.827186Z","iopub.execute_input":"2024-01-22T09:50:02.827945Z","iopub.status.idle":"2024-01-22T09:50:02.834174Z","shell.execute_reply.started":"2024-01-22T09:50:02.827909Z","shell.execute_reply":"2024-01-22T09:50:02.833278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T09:50:04.858493Z","iopub.execute_input":"2024-01-22T09:50:04.858894Z","iopub.status.idle":"2024-01-22T09:50:06.771371Z","shell.execute_reply.started":"2024-01-22T09:50:04.858857Z","shell.execute_reply":"2024-01-22T09:50:06.770582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"code","source":"# 1. Load a Pre-trained ResNet-18 Model\nmodel = models.resnet18(pretrained=True)\n\n# 2. Modify the Final Layer for CIFAR-10\nnum_ftrs = model.fc.in_features\nmodel.fc = nn.Linear(num_ftrs, 10) # CIFAR-10 has 10 classes\n\n# 3. Define Loss Function and Optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n# Assuming you have a GPU to train on\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nwandb.init(project=\"cifar10_resnet18\")\n\n# 4. Training and Validation Functions\ndef train_model(model, criterion, optimizer, num_epochs=25):\n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        print('-' * 10)\n\n        # Each epoch has a training and validation phase\n        for phase in ['train', 'val']:\n            if phase == 'train':\n                model.train()  # Set model to training mode\n            else:\n                model.eval()   # Set model to evaluate mode\n\n            running_loss = 0.0\n            running_corrects = 0\n\n            # Iterate over data.\n            data_iter = train_iter if phase == 'train' else valid_iter\n            for inputs, labels in data_iter:\n                inputs = inputs.to(device)\n                labels = labels.to(device)\n\n                # Zero the parameter gradients\n                optimizer.zero_grad()\n\n                # Forward\n                # Track history if only in train\n                with torch.set_grad_enabled(phase == 'train'):\n                    outputs = model(inputs)\n                    _, preds = torch.max(outputs, 1)\n                    loss = criterion(outputs, labels)\n\n                    # Backward + optimize only if in training phase\n                    if phase == 'train':\n                        loss.backward()\n                        optimizer.step()\n\n                # Statistics\n                running_loss += loss.item() * inputs.size(0)\n                running_corrects += torch.sum(preds == labels.data)\n\n            epoch_loss = running_loss / len(data_iter.dataset)\n            epoch_acc = running_corrects.double() / len(data_iter.dataset)\n\n            if phase == 'train':\n                wandb.log({\"train_loss\": epoch_loss, \"train_acc\": epoch_acc})\n            else:\n                wandb.log({\"val_loss\": epoch_loss, \"val_acc\": epoch_acc})\n            \n            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n\n# 5. Train the Model\ntrain_model(model, criterion, optimizer, num_epochs=25)\n\nwandb.log({\"final_model\": model})\nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:04:45.686326Z","iopub.execute_input":"2024-01-22T10:04:45.687195Z","iopub.status.idle":"2024-01-22T10:21:50.742344Z","shell.execute_reply.started":"2024-01-22T10:04:45.687158Z","shell.execute_reply":"2024-01-22T10:21:50.741629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import csv\n\nlabel_mapping = {0: 'airplane', 1: 'automobile', 2: 'bird', 3: 'cat', 4: 'deer', \n                 5: 'dog', 6: 'frog', 7: 'horse', 8: 'ship', 9: 'truck'}\n\n# Set the model to evaluation mode\nmodel.eval()\n\n# File to save predictions\nsubmission_file = 'submission.csv'\n\n# Write header to the submission file\nwith open(submission_file, 'w', newline='') as file:\n    writer = csv.writer(file)\n    writer.writerow(['id', 'label'])\n\n# Predict and write the results\nwith torch.no_grad():\n    for i, (images, _) in enumerate(test_iter, 1):\n        images = images.to(device)\n        outputs = model(images)\n        _, predicted = torch.max(outputs, 1)\n        predicted_labels = [label_mapping[label] for label in predicted.cpu().numpy()]\n\n        # Write each prediction\n        with open(submission_file, 'a', newline='') as file:\n            writer = csv.writer(file)\n            for idx, label in zip(range(i * batch_size - batch_size + 1, i * batch_size + 1), predicted_labels):\n                writer.writerow([idx, label])\n\nprint(f'Submission file created: {submission_file}')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:59:46.794226Z","iopub.execute_input":"2024-01-22T10:59:46.795171Z","iopub.status.idle":"2024-01-22T11:02:21.491178Z","shell.execute_reply.started":"2024-01-22T10:59:46.795131Z","shell.execute_reply":"2024-01-22T11:02:21.490136Z"},"trusted":true},"execution_count":null,"outputs":[]}]}