{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T10:02:36.391313Z","iopub.execute_input":"2024-01-22T10:02:36.391735Z","iopub.status.idle":"2024-01-22T10:02:36.400845Z","shell.execute_reply.started":"2024-01-22T10:02:36.391703Z","shell.execute_reply":"2024-01-22T10:02:36.399946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:02:36.402541Z","iopub.execute_input":"2024-01-22T10:02:36.402863Z","iopub.status.idle":"2024-01-22T10:02:36.413719Z","shell.execute_reply.started":"2024-01-22T10:02:36.402839Z","shell.execute_reply":"2024-01-22T10:02:36.412852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:02:36.415172Z","iopub.execute_input":"2024-01-22T10:02:36.415448Z","iopub.status.idle":"2024-01-22T10:02:48.627170Z","shell.execute_reply.started":"2024-01-22T10:02:36.415424Z","shell.execute_reply":"2024-01-22T10:02:48.625908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:02:48.628699Z","iopub.execute_input":"2024-01-22T10:02:48.629057Z","iopub.status.idle":"2024-01-22T10:03:46.707952Z","shell.execute_reply.started":"2024-01-22T10:02:48.629013Z","shell.execute_reply":"2024-01-22T10:03:46.706652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:03:46.711278Z","iopub.execute_input":"2024-01-22T10:03:46.711944Z","iopub.status.idle":"2024-01-22T10:23:07.505878Z","shell.execute_reply.started":"2024-01-22T10:03:46.711913Z","shell.execute_reply":"2024-01-22T10:23:07.504427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:23:07.507799Z","iopub.execute_input":"2024-01-22T10:23:07.508191Z","iopub.status.idle":"2024-01-22T10:23:07.513499Z","shell.execute_reply.started":"2024-01-22T10:23:07.508153Z","shell.execute_reply":"2024-01-22T10:23:07.512613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')\nlist_labels =  [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:23:07.515094Z","iopub.execute_input":"2024-01-22T10:23:07.515746Z","iopub.status.idle":"2024-01-22T10:23:07.847053Z","shell.execute_reply.started":"2024-01-22T10:23:07.515707Z","shell.execute_reply":"2024-01-22T10:23:07.846172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:29:22.933511Z","iopub.execute_input":"2024-01-22T10:29:22.934213Z","iopub.status.idle":"2024-01-22T10:29:22.944796Z","shell.execute_reply.started":"2024-01-22T10:29:22.934182Z","shell.execute_reply":"2024-01-22T10:29:22.943769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T10:29:31.421370Z","iopub.execute_input":"2024-01-22T10:29:31.422100Z","iopub.status.idle":"2024-01-22T10:30:34.710375Z","shell.execute_reply.started":"2024-01-22T10:29:31.422069Z","shell.execute_reply":"2024-01-22T10:30:34.709458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"transform_train_sophisticated = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(p=1),\n    torchvision.transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.2),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])\n])\n\ntransform_train = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])\n])\ntransform_train","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:01.419110Z","iopub.execute_input":"2024-01-22T12:43:01.419747Z","iopub.status.idle":"2024-01-22T12:43:01.429527Z","shell.execute_reply.started":"2024-01-22T12:43:01.419711Z","shell.execute_reply":"2024-01-22T12:43:01.428638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_WORKERS = os.cpu_count()\nNUM_WORKERS","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:02.858784Z","iopub.execute_input":"2024-01-22T12:43:02.859769Z","iopub.status.idle":"2024-01-22T12:43:02.865523Z","shell.execute_reply.started":"2024-01-22T12:43:02.859732Z","shell.execute_reply":"2024-01-22T12:43:02.864613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds_augmented, train_valid_ds = [\n    torchvision.datasets.ImageFolder(\n        os.path.join(data_dir, 'train_valid_test', folder),\n        transform=transform_train_sophisticated\n    ) \n    for folder in ['train', 'train_valid']\n]\n\ntrain_ds = torchvision.datasets.ImageFolder(\n        os.path.join(data_dir, 'train_valid_test', \"train\"),\n        transform=transform_train\n    ) \n\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter_augmented, train_valid_iter = [\n    torch.utils.data.DataLoader(dataset, batch_size, shuffle=True, drop_last=True, num_workers=NUM_WORKERS)\n    for dataset in (train_ds_augmented, train_valid_ds)\n]\ntrain_iter = torch.utils.data.DataLoader(train_ds,batch_size, shuffle=True, drop_last=True, num_workers=NUM_WORKERS)\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False, drop_last=True, num_workers=NUM_WORKERS)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:05.390409Z","iopub.execute_input":"2024-01-22T12:43:05.391228Z","iopub.status.idle":"2024-01-22T12:43:07.436377Z","shell.execute_reply.started":"2024-01-22T12:43:05.391192Z","shell.execute_reply":"2024-01-22T12:43:07.435571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds, valid_ds, test_ds","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:09.458709Z","iopub.execute_input":"2024-01-22T12:43:09.459363Z","iopub.status.idle":"2024-01-22T12:43:09.465708Z","shell.execute_reply.started":"2024-01-22T12:43:09.459325Z","shell.execute_reply":"2024-01-22T12:43:09.464776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds[16000][1]","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:10.561724Z","iopub.execute_input":"2024-01-22T12:43:10.562352Z","iopub.status.idle":"2024-01-22T12:43:10.570290Z","shell.execute_reply.started":"2024-01-22T12:43:10.562317Z","shell.execute_reply":"2024-01-22T12:43:10.569363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation techniqu\n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\ndevice","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:12.249504Z","iopub.execute_input":"2024-01-22T12:43:12.250321Z","iopub.status.idle":"2024-01-22T12:43:12.256602Z","shell.execute_reply.started":"2024-01-22T12:43:12.250285Z","shell.execute_reply":"2024-01-22T12:43:12.255586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_net = torchvision.models.resnet18(pretrained=True).to(device)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:12.735019Z","iopub.execute_input":"2024-01-22T12:43:12.735382Z","iopub.status.idle":"2024-01-22T12:43:12.980725Z","shell.execute_reply.started":"2024-01-22T12:43:12.735352Z","shell.execute_reply":"2024-01-22T12:43:12.979878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pretrained_net.fc = nn.Linear(pretrained_net.fc.in_features, 10).to(device)\nnn.init.xavier_normal_(pretrained_net.fc.weight)\nnn.init.constant_(pretrained_net.fc.bias, 0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:13.129540Z","iopub.execute_input":"2024-01-22T12:43:13.129931Z","iopub.status.idle":"2024-01-22T12:43:13.138980Z","shell.execute_reply.started":"2024-01-22T12:43:13.129901Z","shell.execute_reply":"2024-01-22T12:43:13.138142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"criterion = nn.CrossEntropyLoss(label_smoothing=0.1)\noptimizer = torch.optim.AdamW(pretrained_net.parameters(), lr=1e-4, weight_decay=0.0001)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer, 'min', patience=3, verbose=True, min_lr=1e-6, factor=0.5\n)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T12:43:14.054646Z","iopub.execute_input":"2024-01-22T12:43:14.055014Z","iopub.status.idle":"2024-01-22T12:43:14.062144Z","shell.execute_reply.started":"2024-01-22T12:43:14.054984Z","shell.execute_reply":"2024-01-22T12:43:14.061051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm.notebook import trange, tqdm\nimport wandb\n\nepochs = 25\n\nwandb.init(\n    project='CIFAR-10',\n    config={\n        \"epochs\": epochs,\n        \"batch_size\": batch_size,\n        \"optimizer\": type(optimizer),\n        \"transforms\": transform_train_sophisticated,\n    },\n    mode=\"disabled\"\n)\n\nfor epoch in trange(epochs):\n    accurate, total, losses = 0, 0, 0\n    for X, y in tqdm(train_iter):\n        X, y = X.to(device), y.to(device)\n        y_pred = pretrained_net(X)\n        loss = criterion(y_pred, y)\n        _, predicted = torch.max(y_pred, 1)\n        accurate += (y == predicted).sum().float()\n        losses += loss.item()\n        total += len(y)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n    for X, y in tqdm(train_iter_augmented):\n        X, y = X.to(device), y.to(device)\n        y_pred = pretrained_net(X)\n        loss = criterion(y_pred, y)\n        _, predicted = torch.max(y_pred, 1)\n        accurate += (y == predicted).sum().float()\n        losses += loss.item()\n        total += len(y)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step() \n    wandb.log({\n        'train loss': losses / len(train_iter_augmented) / 2,\n        'train accuracy': accurate / total\n    })\n    for X, y in tqdm(valid_iter):\n        X, y = X.to(device), y.to(device)\n        with torch.no_grad():\n            y_pred = pretrained_net(X)\n            loss = criterion(y_pred, y)\n            _, predicted = torch.max(y_pred, 1)\n            accurate += (y == predicted).sum().float()\n            losses += loss.item()\n            total += len(y)\n    wandb.log({\n        'valid loss': losses / len(valid_iter),\n        'valid accuracy': accurate / total\n    })","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:01:13.527838Z","iopub.execute_input":"2024-01-22T13:01:13.528713Z","iopub.status.idle":"2024-01-22T13:36:59.055867Z","shell.execute_reply.started":"2024-01-22T13:01:13.528672Z","shell.execute_reply":"2024-01-22T13:36:59.054727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_labels\ndict_labels = {i: list_labels[i] for i in range(10)}\ndict_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:11:24.221659Z","iopub.execute_input":"2024-01-22T14:11:24.222395Z","iopub.status.idle":"2024-01-22T14:11:24.230596Z","shell.execute_reply.started":"2024-01-22T14:11:24.222353Z","shell.execute_reply":"2024-01-22T14:11:24.229600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = '/kaggle/working/test'\nImageId = sorted([(''.join(filter(str.isdigit, name ))) for name in os.listdir(directory)])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:11:25.947706Z","iopub.execute_input":"2024-01-22T14:11:25.948087Z","iopub.status.idle":"2024-01-22T14:11:26.547440Z","shell.execute_reply.started":"2024-01-22T14:11:25.948056Z","shell.execute_reply":"2024-01-22T14:11:26.546616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    pretrained_net.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_iter):\n        output = pretrained_net(data[0].to(device)).argmax(1)\n        predicted = output.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[ImageId, test_pred.numpy()], columns=['id', 'label']).astype({'id':'int', 'label':'int'})\n    out_df['label'] = out_df['label'].map(dict_labels)\nout_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T14:11:44.293400Z","iopub.execute_input":"2024-01-22T14:11:44.294156Z","iopub.status.idle":"2024-01-22T14:14:26.550768Z","shell.execute_reply.started":"2024-01-22T14:11:44.294122Z","shell.execute_reply":"2024-01-22T14:14:26.549820Z"},"trusted":true},"execution_count":null,"outputs":[]}]}