{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30636,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Alessandro Castelli Kaggle: https://www.kaggle.com/code/alessandromajumba/ex5-ml","metadata":{}},{"cell_type":"markdown","source":"# Kaggle Competition: https://www.kaggle.com/competitions/cifar-10","metadata":{}},{"cell_type":"markdown","source":"# WANDB: https://wandb.ai/ales-2000-09/CIFAR-10?workspace=user-ales-2000-09","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T22:48:51.069190Z","iopub.execute_input":"2024-01-22T22:48:51.069603Z","iopub.status.idle":"2024-01-22T22:48:52.056828Z","shell.execute_reply.started":"2024-01-22T22:48:51.069566Z","shell.execute_reply":"2024-01-22T22:48:52.054871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:52.061192Z","iopub.execute_input":"2024-01-22T22:48:52.063351Z","iopub.status.idle":"2024-01-22T22:48:53.006907Z","shell.execute_reply.started":"2024-01-22T22:48:52.063261Z","shell.execute_reply":"2024-01-22T22:48:53.005889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:53.008308Z","iopub.execute_input":"2024-01-22T22:48:53.008702Z","iopub.status.idle":"2024-01-22T22:48:54.219000Z","shell.execute_reply.started":"2024-01-22T22:48:53.008664Z","shell.execute_reply":"2024-01-22T22:48:54.217882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:54.221434Z","iopub.execute_input":"2024-01-22T22:48:54.221962Z","iopub.status.idle":"2024-01-22T22:48:55.166033Z","shell.execute_reply.started":"2024-01-22T22:48:54.221921Z","shell.execute_reply":"2024-01-22T22:48:55.165006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:55.167395Z","iopub.execute_input":"2024-01-22T22:48:55.167794Z","iopub.status.idle":"2024-01-22T22:48:56.159251Z","shell.execute_reply.started":"2024-01-22T22:48:55.167756Z","shell.execute_reply":"2024-01-22T22:48:56.158247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:56.160586Z","iopub.execute_input":"2024-01-22T22:48:56.160943Z","iopub.status.idle":"2024-01-22T22:48:57.067857Z","shell.execute_reply.started":"2024-01-22T22:48:56.160909Z","shell.execute_reply":"2024-01-22T22:48:57.066878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from kaggle_secrets import UserSecretsClient\nuser_secrets = UserSecretsClient()\nsecret_value_0 = user_secrets.get_secret(\"Label\")\n\nimport wandb\nwandb.login(key=secret_value_0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:57.069211Z","iopub.execute_input":"2024-01-22T22:48:57.069557Z","iopub.status.idle":"2024-01-22T22:48:58.961432Z","shell.execute_reply.started":"2024-01-22T22:48:57.069523Z","shell.execute_reply":"2024-01-22T22:48:58.960523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:58.962635Z","iopub.execute_input":"2024-01-22T22:48:58.962972Z","iopub.status.idle":"2024-01-22T22:48:59.998330Z","shell.execute_reply.started":"2024-01-22T22:48:58.962937Z","shell.execute_reply":"2024-01-22T22:48:59.997367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:48:59.999674Z","iopub.execute_input":"2024-01-22T22:49:00.000049Z","iopub.status.idle":"2024-01-22T22:49:01.151041Z","shell.execute_reply.started":"2024-01-22T22:49:00.000008Z","shell.execute_reply":"2024-01-22T22:49:01.150007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.init(project='CIFAR-10', save_code=True)\nbatch_size = 64\nwandb.log({'Batch_size': batch_size})\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:49:01.154854Z","iopub.execute_input":"2024-01-22T22:49:01.155264Z","iopub.status.idle":"2024-01-22T22:50:47.054860Z","shell.execute_reply.started":"2024-01-22T22:49:01.155224Z","shell.execute_reply":"2024-01-22T22:50:47.053805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    transforms.RandomResizedCrop(32),\n    transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    \n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465],\n                                     [0.2023, 0.1994, 0.2010])])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:50:47.057321Z","iopub.execute_input":"2024-01-22T22:50:47.057632Z","iopub.status.idle":"2024-01-22T22:50:47.063512Z","shell.execute_reply.started":"2024-01-22T22:50:47.057606Z","shell.execute_reply":"2024-01-22T22:50:47.062519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:50:47.065067Z","iopub.execute_input":"2024-01-22T22:50:47.065438Z","iopub.status.idle":"2024-01-22T22:50:49.073248Z","shell.execute_reply.started":"2024-01-22T22:50:47.065403Z","shell.execute_reply":"2024-01-22T22:50:49.072182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load pre-trained ResNet-18 model\nresnet18 = torchvision.models.resnet18(pretrained=True)\n\n# Modify the fully connected layer (fc) for the CIFAR-10 dataset (10 classes)\nresnet18.fc = nn.Linear(resnet18.fc.in_features, 10)\n\n# Initialize the weights of the fully connected layer using Xavier normal initialization\nnn.init.xavier_normal_(resnet18.fc.weight)\n\n# Initialize the bias of the fully connected layer to zeros\nnn.init.constant_(resnet18.fc.bias, 0)\n\n# Set the device to GPU if available, otherwise use CPU\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nresnet18 = resnet18.to(device)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:50:49.074734Z","iopub.execute_input":"2024-01-22T22:50:49.075116Z","iopub.status.idle":"2024-01-22T22:50:49.296352Z","shell.execute_reply.started":"2024-01-22T22:50:49.075078Z","shell.execute_reply":"2024-01-22T22:50:49.295427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lr = 0.003\nwandb.log({'LR': lr})\n\n# Define the loss function and optimizer\ncriterion = nn.CrossEntropyLoss(label_smoothing=0.1)\noptimizer = optim.SGD(resnet18.parameters(), lr=lr, momentum=0.9)\n\n# Training loop\nnum_epochs = 32\nfor epoch in range(num_epochs):\n    resnet18.train()\n    total_loss = 0.0  # Initialize total loss for the epoch\n    for inputs, labels in train_iter:\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = resnet18(inputs)\n        loss = criterion(outputs, labels)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n        \n    average_loss = total_loss / len(train_iter)  # Calculate the average loss for the epoch\n    print(f'Finish Epoch: {epoch + 1}, Training Loss: {average_loss}')\n    wandb.log({'Training Loss': average_loss})\n    wandb.log({'Epoch': epoch + 1})\n\n    # Evaluation on the validation set\n    resnet18.eval()\n    correct = 0\n    total = 0\n    with torch.no_grad():\n        val_loss = 0.0\n        for inputs, labels in valid_iter:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = resnet18(inputs)\n            _, predicted = torch.max(outputs, 1)\n            total += labels.size(0)\n            correct += (predicted == labels).sum().item()\n            loss = criterion(outputs, labels)\n            val_loss += loss.item()\n\n    average_loss2 = val_loss / len(valid_iter)  \n    print(f'Finish Epoch: {epoch + 1}, Validation Loss: {average_loss2}')\n    wandb.log({'Validation Loss': average_loss2})\n\n    accuracy = correct / total\n    print(f'Validation Accuracy: {accuracy}')\n    wandb.log({'Accuracy on validation': accuracy})\n    \nwandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T22:50:49.297469Z","iopub.execute_input":"2024-01-22T22:50:49.297773Z","iopub.status.idle":"2024-01-22T23:15:05.739528Z","shell.execute_reply.started":"2024-01-22T22:50:49.297747Z","shell.execute_reply":"2024-01-22T23:15:05.738812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"resnet18 = resnet18.to(device)\nresnet18.eval()\npredictions = []  # List to store predicted labels\n\nwith torch.no_grad():\n    for inputs, labels in test_iter:\n        inputs, labels = inputs.to(device), labels.to(device)\n        outputs = resnet18(inputs)\n        predicted = outputs.argmax(dim=1)\n        predictions.extend(predicted.type(torch.int32).cpu().numpy())\n\n# Create submission file\nidentifiers = list(range(1, len(test_ds) + 1))\nidentifiers.sort(key=lambda x: str(x))\nsubmission_df = pd.DataFrame({'id': identifiers, 'label': predictions})\nsubmission_df['label'] = submission_df['label'].apply(lambda x: train_ds.classes[x])\nsubmission_df.to_csv('submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:15:05.740535Z","iopub.execute_input":"2024-01-22T23:15:05.740789Z","iopub.status.idle":"2024-01-22T23:17:46.426826Z","shell.execute_reply.started":"2024-01-22T23:15:05.740766Z","shell.execute_reply":"2024-01-22T23:17:46.425921Z"},"trusted":true},"execution_count":null,"outputs":[]}]}