{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-22T11:24:45.164353Z","iopub.execute_input":"2024-01-22T11:24:45.164683Z","iopub.status.idle":"2024-01-22T11:24:46.011188Z","shell.execute_reply.started":"2024-01-22T11:24:45.164650Z","shell.execute_reply":"2024-01-22T11:24:46.010255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\n\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:24:46.015712Z","iopub.execute_input":"2024-01-22T11:24:46.016519Z","iopub.status.idle":"2024-01-22T11:24:50.227739Z","shell.execute_reply.started":"2024-01-22T11:24:46.016482Z","shell.execute_reply":"2024-01-22T11:24:50.226897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:24:50.228788Z","iopub.execute_input":"2024-01-22T11:24:50.229184Z","iopub.status.idle":"2024-01-22T11:25:05.439010Z","shell.execute_reply.started":"2024-01-22T11:24:50.229158Z","shell.execute_reply":"2024-01-22T11:25:05.437880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:25:05.440849Z","iopub.execute_input":"2024-01-22T11:25:05.441199Z","iopub.status.idle":"2024-01-22T11:26:03.108843Z","shell.execute_reply.started":"2024-01-22T11:25:05.441171Z","shell.execute_reply":"2024-01-22T11:26:03.107712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:26:03.110565Z","iopub.execute_input":"2024-01-22T11:26:03.110960Z","iopub.status.idle":"2024-01-22T11:45:05.416818Z","shell.execute_reply.started":"2024-01-22T11:26:03.110925Z","shell.execute_reply":"2024-01-22T11:45:05.415425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '/kaggle/working/'","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:45:05.421077Z","iopub.execute_input":"2024-01-22T11:45:05.421422Z","iopub.status.idle":"2024-01-22T11:45:05.426107Z","shell.execute_reply.started":"2024-01-22T11:45:05.421394Z","shell.execute_reply":"2024-01-22T11:45:05.425299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:45:05.427178Z","iopub.execute_input":"2024-01-22T11:45:05.427408Z","iopub.status.idle":"2024-01-22T11:45:05.557429Z","shell.execute_reply.started":"2024-01-22T11:45:05.427388Z","shell.execute_reply":"2024-01-22T11:45:05.556470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:45:05.558703Z","iopub.execute_input":"2024-01-22T11:45:05.559012Z","iopub.status.idle":"2024-01-22T11:45:05.569630Z","shell.execute_reply.started":"2024-01-22T11:45:05.558986Z","shell.execute_reply":"2024-01-22T11:45:05.568676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:45:05.570939Z","iopub.execute_input":"2024-01-22T11:45:05.571571Z","iopub.status.idle":"2024-01-22T11:46:08.107085Z","shell.execute_reply.started":"2024-01-22T11:45:05.571521Z","shell.execute_reply":"2024-01-22T11:46:08.106062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"weights = torchvision.models.ResNet18_Weights.DEFAULT\npreprocess = weights.transforms()\n\ntransform_train = torchvision.transforms.Compose([\n    transforms.RandomCrop(32, padding=4),\n    transforms.RandomHorizontalFlip(p=0.4),\n    preprocess\n  ])\n\ntransform_test = torchvision.transforms.Compose([\n    preprocess])","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:48:12.523121Z","iopub.execute_input":"2024-01-22T11:48:12.523476Z","iopub.status.idle":"2024-01-22T11:48:12.529366Z","shell.execute_reply.started":"2024-01-22T11:48:12.523448Z","shell.execute_reply":"2024-01-22T11:48:12.528343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"transform_train","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:48:17.008404Z","iopub.execute_input":"2024-01-22T11:48:17.008817Z","iopub.status.idle":"2024-01-22T11:48:17.015962Z","shell.execute_reply.started":"2024-01-22T11:48:17.008786Z","shell.execute_reply":"2024-01-22T11:48:17.014872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:48:21.789086Z","iopub.execute_input":"2024-01-22T11:48:21.789451Z","iopub.status.idle":"2024-01-22T11:48:23.663535Z","shell.execute_reply.started":"2024-01-22T11:48:21.789421Z","shell.execute_reply":"2024-01-22T11:48:23.662652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features, train_labels = next(iter(train_iter))","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:50:18.930697Z","iopub.execute_input":"2024-01-22T11:50:18.931629Z","iopub.status.idle":"2024-01-22T11:50:19.183676Z","shell.execute_reply.started":"2024-01-22T11:50:18.931581Z","shell.execute_reply":"2024-01-22T11:50:19.182816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:50:28.660369Z","iopub.execute_input":"2024-01-22T11:50:28.661260Z","iopub.status.idle":"2024-01-22T11:50:28.668595Z","shell.execute_reply.started":"2024-01-22T11:50:28.661223Z","shell.execute_reply":"2024-01-22T11:50:28.667686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"code","source":"resnet18model = torchvision.models.resnet18(weights=torchvision.models.ResNet18_Weights.DEFAULT)\n\nnum_classes = 10\nresnet18model.fc = nn.Sequential(\n    nn.Linear(resnet18model.fc.in_features, num_classes),\n    nn.Softmax(dim=1)\n)\nnn.init.xavier_normal_(resnet18model.fc[0].weight)\nnn.init.constant_(resnet18model.fc[0].bias, 0)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:54:07.270732Z","iopub.execute_input":"2024-01-22T11:54:07.271109Z","iopub.status.idle":"2024-01-22T11:54:07.510751Z","shell.execute_reply.started":"2024-01-22T11:54:07.271076Z","shell.execute_reply":"2024-01-22T11:54:07.509678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Putting the model on GPU","metadata":{}},{"cell_type":"code","source":"device = 'cuda' if torch.cuda.is_available() else 'cpu'\n \nresnet18model.to(device)\n\nd = next(resnet18model.parameters()).device\n\nif d.type == 'cuda':\n    print(\"Model is on GPU\")\nelse:\n    print(\"Model is on CPU\")\n    resnet18model = torch.nn.DataParallel(resnet18model) # if multiple GPUs use them","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:54:41.028446Z","iopub.execute_input":"2024-01-22T11:54:41.029280Z","iopub.status.idle":"2024-01-22T11:54:41.201569Z","shell.execute_reply.started":"2024-01-22T11:54:41.029249Z","shell.execute_reply":"2024-01-22T11:54:41.200647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def my_accuracy(predictions, labels):\n    #print(predictions)\n    predictions = torch.argmax(predictions,dim=1)\n    #print(predictions)\n    #print(labels)\n    correct = (predictions == labels)\n    #print(correct)\n    acc = sum(correct) / len(predictions)\n    #print(acc.item())\n    return acc.item()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T11:54:43.439563Z","iopub.execute_input":"2024-01-22T11:54:43.440240Z","iopub.status.idle":"2024-01-22T11:54:43.445175Z","shell.execute_reply.started":"2024-01-22T11:54:43.440211Z","shell.execute_reply":"2024-01-22T11:54:43.444277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import wandb\nwandb.init(project='lab_cifar10', save_code=True, config={\n            \"epochs\": 10,\n            \"lr\": 3e-3,mode=\"disabled\"\n            }) # mode=\"disable\"\nresnet18model\n","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:01:26.865892Z","iopub.execute_input":"2024-01-22T13:01:26.866253Z","iopub.status.idle":"2024-01-22T13:02:03.291569Z","shell.execute_reply.started":"2024-01-22T13:01:26.866228Z","shell.execute_reply":"2024-01-22T13:02:03.290666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nconfig = wandb.config\n\nnum_epochs = config.epochs # should be more than enough, but can be changed\nlr = config.lr # e.q to 0.007, you can change it if needed\n\n# to be adjusted\ncriterion = nn.CrossEntropyLoss()\noptimizer = torch.optim.SGD(resnet18model.parameters(), lr=lr)\nscheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=3, verbose=True, min_lr=7e-5, factor=0.5)\n\nfor epoch in range(num_epochs):\n    training_loss_epoch = 0\n    training_accuracy_epoch = 0\n    cnt  = 0\n    for train_features_batch, train_labels_batch in train_iter:\n        train_features_batch, train_labels_batch = train_features_batch.to(device), train_labels_batch.to(device)\n        optimizer.zero_grad() # please don't forget!\n        \n        y_hat = resnet18model(train_features_batch)\n        #train_labels_batch = train_labels_batch.reshape(len(train_labels_batch),1)\n        #train_labels_batch = train_labels_batch.float()\n        \n        training_loss = criterion(y_hat, train_labels_batch)\n        training_loss.backward() # remember: You need to tell w.r.t what the gradient is computed\n        optimizer.step()\n        \n        training_loss_epoch += training_loss.item()\n        \n        #print(training_loss)\n        #wandb.log({'training_loss_batch': training_loss.item()})\n        \n        train_accuracy = my_accuracy(y_hat, train_labels_batch)\n        training_accuracy_epoch += train_accuracy\n        #print(\"train iteration\" +str(cnt))\n        cnt+=1\n        \n    validation_loss_epoch = 0\n    validation_accuracy_epoch = 0\n    cnt = 0\n    for val_features_batch, val_labels_batch in valid_iter:\n        val_features_batch, val_labels_batch = val_features_batch.to(device), val_labels_batch.to(device)\n        \n        with torch.no_grad():\n            #reshaped_val_features_batch = val_features_batch.reshape(len(val_features_batch), -1)\n        \n            y_hat_val = resnet18model(val_features_batch)\n            #val_labels_batch = val_labels_batch.reshape(len(val_labels_batch),1)\n            #val_labels_batch = val_labels_batch.float()\n            \n            validation_loss = criterion(y_hat_val, val_labels_batch)\n            validation_loss_epoch += validation_loss.item()\n            \n            val_accuracy = my_accuracy(y_hat_val, val_labels_batch)\n            validation_accuracy_epoch += val_accuracy\n            #wandb.log({'validation_loss_batch': validation_loss.item()})\n            #print(\"val iteration\" +str(cnt))\n            cnt+=1\n    \n    wandb.log({'training_loss_epoch': training_loss_epoch/len(train_iter), 'validation_loss_epoch': validation_loss_epoch/len(valid_iter)})\n    wandb.log({'training_accuracy_epoch': training_accuracy_epoch/len(train_iter), 'validation_accuracy_epoch': validation_accuracy_epoch/len(valid_iter)})\n    print({'training_loss_epoch': training_loss_epoch/len(train_iter), 'validation_loss_epoch': validation_loss_epoch/len(valid_iter)})\n    print({'training_accuracy_epoch': training_accuracy_epoch/len(train_iter), 'validation_accuracy_epoch': validation_accuracy_epoch/len(valid_iter)})\n    if (training_loss_epoch/len(train_iter) - validation_loss_epoch/len(valid_iter) < -0.08) and training_loss_epoch/len(train_iter) - validation_loss_epoch/len(valid_iter) < -0.1:\n        print('Early stopping!')\n        break","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:02:14.658361Z","iopub.execute_input":"2024-01-22T13:02:14.659166Z","iopub.status.idle":"2024-01-22T13:32:36.361220Z","shell.execute_reply.started":"2024-01-22T13:02:14.659125Z","shell.execute_reply":"2024-01-22T13:32:36.360320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wandb.finish()","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:25.331701Z","iopub.execute_input":"2024-01-22T13:33:25.332420Z","iopub.status.idle":"2024-01-22T13:33:31.543770Z","shell.execute_reply.started":"2024-01-22T13:33:25.332387Z","shell.execute_reply":"2024-01-22T13:33:31.543021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.no_grad():\n    resnet18model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_iter):\n        test_batch = data[0].to(device)\n        output = resnet18model(test_batch)\n        #, predicted = torch.max(output.data, 1)\n        predicted = output\n        predicted = predicted.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    predictions = torch.argmax(test_pred,dim=1)\n    out_df = pd.DataFrame(np.c_[np.arange(1, predictions.shape[0]+1)[:,None], predictions.numpy()], columns=['id', 'label'])\n    out_df['label'] = out_df['label'].map({0:'airplane', 1:'automobile', 2:'bird', 3:'cat', 4:'deer', 5:'dog', 6:'frog', 7:'horse', 8:'ship', 9:'truck'})\n    out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T13:33:40.779157Z","iopub.execute_input":"2024-01-22T13:33:40.779518Z","iopub.status.idle":"2024-01-22T13:46:51.420410Z","shell.execute_reply.started":"2024-01-22T13:33:40.779489Z","shell.execute_reply":"2024-01-22T13:46:51.419290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test_pred[:3])\na = torch.argmax(test_pred[:3],dim=1)\nprint(a)\n\na = os.listdir('/kaggle/working/test')[:10]\na\n\n#for im in a:\n #   im = plt.imread('/kaggle/working/test/' + im)\n #   plt.imshow(im)\n #   plt.show()#","metadata":{"execution":{"iopub.status.busy":"2024-01-22T20:06:39.078600Z","iopub.execute_input":"2024-01-22T20:06:39.078927Z"},"trusted":true},"execution_count":null,"outputs":[]}]}