{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[]},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"b0M9dabft2Xq","execution":{"iopub.status.busy":"2024-01-22T23:32:43.17713Z","iopub.execute_input":"2024-01-22T23:32:43.177571Z","iopub.status.idle":"2024-01-22T23:32:43.493629Z","shell.execute_reply.started":"2024-01-22T23:32:43.177541Z","shell.execute_reply":"2024-01-22T23:32:43.492598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{"id":"tQ3qQuclt2Xt"}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"id":"Lra4_BMrt2Xu","execution":{"iopub.status.busy":"2024-01-22T23:32:53.689636Z","iopub.execute_input":"2024-01-22T23:32:53.690019Z","iopub.status.idle":"2024-01-22T23:32:53.699769Z","shell.execute_reply.started":"2024-01-22T23:32:53.689989Z","shell.execute_reply":"2024-01-22T23:32:53.698697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{"id":"S16RpVHPt2Xu"}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"id":"j-qjmta1t2Xu","outputId":"fc6aa8e8-6c8f-4ab8-ca7f-8ffcf9f2fd5e","execution":{"iopub.status.busy":"2024-01-22T23:32:56.125671Z","iopub.execute_input":"2024-01-22T23:32:56.12643Z","iopub.status.idle":"2024-01-22T23:33:12.160338Z","shell.execute_reply.started":"2024-01-22T23:32:56.126396Z","shell.execute_reply":"2024-01-22T23:33:12.159191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{"id":"DE4QVCXvt2Xu"}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"id":"aQ0YF4n4t2Xu","outputId":"4ac4f2e7-af28-4f69-cc71-e0c5564cc088","execution":{"iopub.status.busy":"2024-01-22T23:33:20.355005Z","iopub.execute_input":"2024-01-22T23:33:20.355404Z","iopub.status.idle":"2024-01-22T23:34:20.56711Z","shell.execute_reply.started":"2024-01-22T23:33:20.355373Z","shell.execute_reply":"2024-01-22T23:34:20.565817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"id":"0ETjW2FBt2Xv","outputId":"90072f44-b56d-45c3-bcd8-35615f606667","execution":{"iopub.status.busy":"2024-01-22T23:34:20.569728Z","iopub.execute_input":"2024-01-22T23:34:20.570161Z","iopub.status.idle":"2024-01-22T23:54:07.094897Z","shell.execute_reply.started":"2024-01-22T23:34:20.570104Z","shell.execute_reply":"2024-01-22T23:54:07.093422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndata_dir = '/kaggle/working/'","metadata":{"id":"VVLoeoMat2Xv","execution":{"iopub.status.busy":"2024-01-22T23:54:43.072763Z","iopub.execute_input":"2024-01-22T23:54:43.073167Z","iopub.status.idle":"2024-01-22T23:54:43.079559Z","shell.execute_reply.started":"2024-01-22T23:54:43.073124Z","shell.execute_reply":"2024-01-22T23:54:43.078488Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"id":"1V3TnpVvt2Xv","outputId":"53141a1c-5735-463d-a210-c5668b6bd768","execution":{"iopub.status.busy":"2024-01-22T23:54:58.980785Z","iopub.execute_input":"2024-01-22T23:54:58.981174Z","iopub.status.idle":"2024-01-22T23:54:59.282814Z","shell.execute_reply.started":"2024-01-22T23:54:58.981122Z","shell.execute_reply":"2024-01-22T23:54:59.281793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n\ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"id":"oYEV3eqnt2Xv","execution":{"iopub.status.busy":"2024-01-22T23:55:03.66193Z","iopub.execute_input":"2024-01-22T23:55:03.662333Z","iopub.status.idle":"2024-01-22T23:55:03.676817Z","shell.execute_reply.started":"2024-01-22T23:55:03.662303Z","shell.execute_reply":"2024-01-22T23:55:03.67561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)\n","metadata":{"id":"cG7jjWaBt2Xw","execution":{"iopub.status.busy":"2024-01-22T23:55:08.375819Z","iopub.execute_input":"2024-01-22T23:55:08.376551Z","iopub.status.idle":"2024-01-22T23:56:11.711411Z","shell.execute_reply.started":"2024-01-22T23:55:08.376519Z","shell.execute_reply":"2024-01-22T23:56:11.710367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours!\n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{"id":"bOnU3Wp0t2Xw"}},{"cell_type":"code","source":"# TODO: Check the transformations in the pre-trained ResNet and sync with yours!\n# TODO: Consider adding more transformations, like HorizontalFlip, and RandomSizeCrop.\ntransform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(), \n    torchvision.transforms.RandomResizedCrop(32), \n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n","metadata":{"id":"WLKOGr94t2Xw","execution":{"iopub.status.busy":"2024-01-23T00:39:16.161012Z","iopub.execute_input":"2024-01-23T00:39:16.161954Z","iopub.status.idle":"2024-01-23T00:39:16.168759Z","shell.execute_reply.started":"2024-01-23T00:39:16.161921Z","shell.execute_reply":"2024-01-23T00:39:16.167704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"id":"13gVOr5-t2Xw","execution":{"iopub.status.busy":"2024-01-23T00:39:18.812601Z","iopub.execute_input":"2024-01-23T00:39:18.813818Z","iopub.status.idle":"2024-01-23T00:39:20.81661Z","shell.execute_reply.started":"2024-01-23T00:39:18.81378Z","shell.execute_reply":"2024-01-23T00:39:20.815755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO:\n* Define the data augmentation technique\n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{"id":"esTT1b1-t2Xx"}},{"cell_type":"code","source":"# TODO: Define the data augmentation technique\ndata_augmentation = torchvision.transforms.Compose([\n    torchvision.transforms.RandomRotation(degrees=15),\n    torchvision.transforms.RandomVerticalFlip(),\n    # Add more augmentation techniques if needed\n])","metadata":{"execution":{"iopub.status.busy":"2024-01-23T01:01:16.440281Z","iopub.execute_input":"2024-01-23T01:01:16.440703Z","iopub.status.idle":"2024-01-23T01:01:16.447174Z","shell.execute_reply.started":"2024-01-23T01:01:16.440676Z","shell.execute_reply":"2024-01-23T01:01:16.446106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\n\n# Initialize WandB\nwandb.init(project=\"cifar-10-finetuning\", name=\"resnet18-finetuning\")\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T01:21:20.057224Z","iopub.execute_input":"2024-01-23T01:21:20.058184Z","iopub.status.idle":"2024-01-23T01:21:56.430924Z","shell.execute_reply.started":"2024-01-23T01:21:20.058135Z","shell.execute_reply":"2024-01-23T01:21:56.429951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torchvision.models as models\nfrom torch.utils.data import DataLoader\nfrom torch.optim.lr_scheduler import StepLR\n\n# Load pre-trained ResNet-18 model\nmodel = models.resnet18(pretrained=True)\n\n# Modify the last fully connected layer for CIFAR-10 (10 classes)\nnum_ftrs = model.fc.in_features\nmodel.fc = nn.Linear(num_ftrs, 10)\n\n# Send the model to GPU if available\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel = model.to(device)\n\n# Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.003, momentum=0.9)\n\n# Fine-tuning parameters\nnum_epochs = 20\n\n# Training loop\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    for inputs, labels in train_iter:\n        inputs, labels = inputs.to(device), labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item()\n    \n\n    # Log training loss for each epoch to WandB\n    wandb.log({\"Training Loss\": running_loss / len(train_iter)})\n\n    # Evaluate on the validation set\n    model.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        for inputs, labels in valid_iter:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n\n    validation_accuracy = correct / total\n\n    # Log validation accuracy for each epoch to WandB\n    wandb.log({\"Validation Accuracy\": validation_accuracy * 100})\n\n    # Print training loss and validation accuracy for each epoch\n    print(f'Epoch {epoch + 1}/{num_epochs}, Loss: {running_loss / len(train_iter)}, '\n          f'Validation Accuracy: {validation_accuracy * 100:.2f}%')\n\nmodel.eval()\nall_predictions = []\n\nwith torch.no_grad():\n    for inputs, _ in test_iter:\n        inputs = inputs.to(device)\n        outputs = model(inputs)\n        _, predicted = torch.max(outputs, 1)\n        all_predictions.extend(predicted.cpu().numpy())\n\n","metadata":{"id":"IjhSu1XMvBBg","execution":{"iopub.status.busy":"2024-01-23T01:22:03.029968Z","iopub.execute_input":"2024-01-23T01:22:03.030801Z","iopub.status.idle":"2024-01-23T01:42:37.168451Z","shell.execute_reply.started":"2024-01-23T01:22:03.030765Z","shell.execute_reply":"2024-01-23T01:42:37.167391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\n\nmodel.eval()\nwith torch.no_grad():\n    for X, _ in test_iter:\n        X = X.to(device)\n        preds.extend(model(X).argmax(dim=1).type(torch.int32).cpu().numpy())\nids = list(range(1, len(test_ds)+1))\nids.sort(key=lambda x: str(x))\ndf = pd.DataFrame({'id': ids, 'label': preds})\ndf['label'] = df['label'].apply(lambda x: train_ds.classes[x])\ndf.to_csv('submission.csv', index=False)\n\n# Finish the WandB run\nwandb.finish()\n\nprint('Training, evaluation, and creation of submission file completed successfully.')","metadata":{"execution":{"iopub.status.busy":"2024-01-23T01:45:31.688024Z","iopub.execute_input":"2024-01-23T01:45:31.688925Z","iopub.status.idle":"2024-01-23T01:48:22.696752Z","shell.execute_reply.started":"2024-01-23T01:45:31.688885Z","shell.execute_reply":"2024-01-23T01:48:22.695761Z"},"trusted":true},"execution_count":null,"outputs":[]}]}