{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"My Kaggle Link : https://www.kaggle.com/code/revathimuthukrishnan/object-recognition\n\nMy Wandb Link : https://wandb.ai/revathimuthukrishnan/Object%20Recognition","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n#Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2024-01-23T08:51:15.728591Z","iopub.execute_input":"2024-01-23T08:51:15.729360Z","iopub.status.idle":"2024-01-23T08:51:16.089688Z","shell.execute_reply.started":"2024-01-23T08:51:15.729323Z","shell.execute_reply":"2024-01-23T08:51:16.088653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import everything needed","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\nimport time\nimport wandb","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:51:16.091486Z","iopub.execute_input":"2024-01-23T08:51:16.091831Z","iopub.status.idle":"2024-01-23T08:51:18.536621Z","shell.execute_reply.started":"2024-01-23T08:51:16.091807Z","shell.execute_reply":"2024-01-23T08:51:18.535866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Unzip datasets","metadata":{}},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:51:18.537988Z","iopub.execute_input":"2024-01-23T08:51:18.538247Z","iopub.status.idle":"2024-01-23T08:51:30.698229Z","shell.execute_reply.started":"2024-01-23T08:51:18.538223Z","shell.execute_reply":"2024-01-23T08:51:30.697206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# WARNING: It can take a lot of time to uncompress!","metadata":{}},{"cell_type":"code","source":"stime = time.time()\n!python -m py7zr x /kaggle/input/cifar-10/train.7z\netime = time.time()\ncompletiontime = etime - stime\nprint(completiontime)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:51:30.699852Z","iopub.execute_input":"2024-01-23T08:51:30.700252Z","iopub.status.idle":"2024-01-23T08:52:29.326977Z","shell.execute_reply.started":"2024-01-23T08:51:30.700215Z","shell.execute_reply":"2024-01-23T08:52:29.325716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stime1 = time.time()\n!python -m py7zr x /kaggle/input/cifar-10/test.7z\netime1 = time.time()\ncompletiontime1 = etime1 - stime1\nprint(completiontime1)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T08:52:29.330220Z","iopub.execute_input":"2024-01-23T08:52:29.330557Z","iopub.status.idle":"2024-01-23T09:11:28.348194Z","shell.execute_reply.started":"2024-01-23T08:52:29.330526Z","shell.execute_reply":"2024-01-23T09:11:28.346951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Params Initialization","metadata":{}},{"cell_type":"code","source":"PARAMS = {\n    'data_dir': '/kaggle/working/',\n    'num_epochs': 25,\n    'batch_size': 64,\n    'valid_ratio': 0.1,\n    'learning_rate': 0.0001,\n    'optimizer': 'adam',  # 'adam', 'rmsprop', 'sgd', etc.\n    'loss_function': 'cross_entropy', # 'cross_entropy','BCE'\n    'momentum': 0,  # Add momentum for SGD optimizer\n    'model_architecture': 'resnet18',  # Change to any other model\n    'augmentation_params': {\n        'size': (32, 32),\n        'scale': (0.8, 1.0),\n        'brightness': 0.2,\n        'contrast': 0.2,\n        'saturation': 0.2,\n        'hue': 0.1,\n        'degree': 10,\n        'normalize_mean': [0.4914, 0.4822, 0.4465], # [0.485, 0.456, 0.406],[0.4914, 0.4822, 0.4465]\n        'normalize_std': [0.2023, 0.1994, 0.2010], #[0.229, 0.224, 0.225],[0.2023, 0.1994, 0.2010]\n    },\n}\n\nwandb.init(project='Object Recognition', save_code=True, config=PARAMS)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:11:28.349906Z","iopub.execute_input":"2024-01-23T09:11:28.350355Z","iopub.status.idle":"2024-01-23T09:16:53.304920Z","shell.execute_reply.started":"2024-01-23T09:11:28.350308Z","shell.execute_reply":"2024-01-23T09:16:53.303985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = PARAMS['data_dir']\nvalid_ratio = PARAMS['valid_ratio']\nbatch_size = PARAMS['batch_size']","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:16:53.306198Z","iopub.execute_input":"2024-01-23T09:16:53.306563Z","iopub.status.idle":"2024-01-23T09:16:53.711168Z","shell.execute_reply.started":"2024-01-23T09:16:53.306526Z","shell.execute_reply":"2024-01-23T09:16:53.710272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')\nprint(type(labels))","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:16:53.712396Z","iopub.execute_input":"2024-01-23T09:16:53.712759Z","iopub.status.idle":"2024-01-23T09:16:54.364129Z","shell.execute_reply.started":"2024-01-23T09:16:53.712726Z","shell.execute_reply":"2024-01-23T09:16:54.363137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    reorg_test(data_dir)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:16:54.365399Z","iopub.execute_input":"2024-01-23T09:16:54.365704Z","iopub.status.idle":"2024-01-23T09:16:54.900408Z","shell.execute_reply.started":"2024-01-23T09:16:54.365676Z","shell.execute_reply":"2024-01-23T09:16:54.899327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:16:54.902761Z","iopub.execute_input":"2024-01-23T09:16:54.903792Z","iopub.status.idle":"2024-01-23T09:17:56.744558Z","shell.execute_reply.started":"2024-01-23T09:16:54.903753Z","shell.execute_reply":"2024-01-23T09:17:56.743689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transformations\n- Pre-trained networks usually apply some transformations to their inputs\n- TODOs:\n    - Check the transformations in the pre-trained ResNet and sync with yours! \n    - Consider adding more transformations, like HorizontalFlip, mnRandomSizeCrop","metadata":{}},{"cell_type":"code","source":"transform_train = torchvision.transforms.Compose([\n    torchvision.transforms.RandomHorizontalFlip(),\n    torchvision.transforms.ColorJitter(\n        brightness=PARAMS['augmentation_params']['brightness'],\n        contrast=PARAMS['augmentation_params']['contrast'],\n        saturation=PARAMS['augmentation_params']['saturation'],\n        hue=PARAMS['augmentation_params']['hue']),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(\n        mean=PARAMS['augmentation_params']['normalize_mean'],\n        std=PARAMS['augmentation_params']['normalize_std'])\n])\n\ntransform_test = torchvision.transforms.Compose([\n    #torchvision.transforms.Resize(32),\n    torchvision.transforms.ToTensor(),\n    torchvision.transforms.Normalize(\n        mean=PARAMS['augmentation_params']['normalize_mean'],\n        std=PARAMS['augmentation_params']['normalize_std'])\n])\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:17:56.746026Z","iopub.execute_input":"2024-01-23T09:17:56.746372Z","iopub.status.idle":"2024-01-23T09:17:57.167857Z","shell.execute_reply.started":"2024-01-23T09:17:56.746334Z","shell.execute_reply":"2024-01-23T09:17:57.166992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:17:57.169750Z","iopub.execute_input":"2024-01-23T09:17:57.170129Z","iopub.status.idle":"2024-01-23T09:17:59.392280Z","shell.execute_reply.started":"2024-01-23T09:17:57.170092Z","shell.execute_reply":"2024-01-23T09:17:59.391211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:17:59.393721Z","iopub.execute_input":"2024-01-23T09:17:59.394082Z","iopub.status.idle":"2024-01-23T09:17:59.842547Z","shell.execute_reply.started":"2024-01-23T09:17:59.394044Z","shell.execute_reply":"2024-01-23T09:17:59.841572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nprint(f\"Using device: {device}\")","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:17:59.847005Z","iopub.execute_input":"2024-01-23T09:17:59.847610Z","iopub.status.idle":"2024-01-23T09:18:00.429029Z","shell.execute_reply.started":"2024-01-23T09:17:59.847567Z","shell.execute_reply":"2024-01-23T09:18:00.428063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## TODO: \n* Define the data augmentation technique \n* Fine-tune a pre-trained ResNet-18 on CIFAR-10\n* Submit your submission file to Kaggle (as described in the competition rules) and get your accuracy on the test set","metadata":{}},{"cell_type":"markdown","source":"ResNet18 Model with Weight","metadata":{}},{"cell_type":"code","source":"model_name = PARAMS['model_architecture']\n\nif model_name in dir(torchvision.models):\n    model_function = getattr(torchvision.models, model_name)\n    model = model_function(pretrained=True)\nelse:\n    raise ValueError(f\"Unsupported model architecture: {model_name}\")\n    \nmodel.fc = nn.Linear(model.fc.in_features, 10)\nnn.init.xavier_normal_(model.fc.weight)\nnn.init.constant_(model.fc.bias, 0)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:18:00.430335Z","iopub.execute_input":"2024-01-23T09:18:00.430698Z","iopub.status.idle":"2024-01-23T09:18:01.994820Z","shell.execute_reply.started":"2024-01-23T09:18:00.430666Z","shell.execute_reply":"2024-01-23T09:18:01.993975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Optimizer and Loss Functions","metadata":{}},{"cell_type":"code","source":"# Update the loss function based on the PARAMS\nif PARAMS['loss_function'] == 'cross_entropy':\n    criterion = nn.CrossEntropyLoss()\nelif PARAMS['loss_function'] == 'BCE':\n    criterion = nn.BCEWithLogitsLoss()\n# Add more conditions for other loss functions as needed\nelse:\n    raise ValueError(f\"Unsupported loss function: {PARAMS['loss_function']}\")\n\n# Update the optimizer based on the PARAMS\nif PARAMS['optimizer'] == 'adam':\n    optimizer = optim.Adam(model.parameters(), lr=PARAMS['learning_rate'])\nelif PARAMS['optimizer'] == 'rmsprop':\n    optimizer = optim.RMSprop(model.parameters(), lr=PARAMS['learning_rate'])\nelif PARAMS['optimizer'] == 'sgd':\n    optimizer = optim.SGD(model.parameters(), lr=PARAMS['learning_rate'], momentum=PARAMS['momentum'])\nelse:\n    raise ValueError(f\"Unsupported optimizer: {PARAMS['optimizer']}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:18:01.996128Z","iopub.execute_input":"2024-01-23T09:18:01.996451Z","iopub.status.idle":"2024-01-23T09:18:02.370857Z","shell.execute_reply.started":"2024-01-23T09:18:01.996418Z","shell.execute_reply":"2024-01-23T09:18:02.369941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training Loop","metadata":{}},{"cell_type":"code","source":"# Define loss function and optimizer\n#criterion = nn.CrossEntropyLoss()\n#optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n# Train the model\nnum_epochs = PARAMS['num_epochs']\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    correct_train = 0\n    total_train = 0\n    start_time = time.time()     \n    print(f'Epoch [{epoch + 1}/{num_epochs}]')\n    for inputs, labels in train_iter:\n        inputs, labels = inputs.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        # Perform optimizer step\n        optimizer.step()    \n          # Update running_loss\n        running_loss += loss.item()\n        # Calculate accuracy\n        _, predicted = torch.max(outputs.data, 1)\n        total_train += labels.size(0)\n        correct_train += (predicted == labels).sum().item()\n\n    # Print training loss at the end of each epoch\n    end_time = time.time()  # Record the end time for the epoch\n    epoch_time = end_time - start_time  # Calculate the time taken for the epoch\n    average_loss = running_loss / len(train_iter)\n    train_accuracy = correct_train / total_train  \n    # Append training metrics to lists\n    #train_losses.append(average_loss)\n    #train_accuracies.append(train_accuracy)\n    # Log training accuracy and loss to WandB\n    wandb.log({\"train_loss\": average_loss, \"train_accuracy\": train_accuracy})\n    print(f'Training Loss: {average_loss}, Training Accuracy: {train_accuracy},Time: {epoch_time} seconds')\n    \n    # Validate the model\n    model.eval()\n    correct_val = 0\n    total_val = 0\n    val_running_loss = 0.0    \n    start_val_time = time.time()\n    with torch.no_grad():\n        for inputs, labels in valid_iter:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)            \n            val_loss = criterion(outputs, labels)\n            val_running_loss += val_loss.item()\n            _, predicted = torch.max(outputs.data, 1)\n            total_val += labels.size(0)\n            correct_val += (predicted == labels).sum().item()\n\n     # Print validation loss and accuracy\n    validation_loss = val_running_loss / len(valid_iter)\n    validation_accuracy = correct_val / total_val\n    end_val_time = time.time()  # Record the end time for validation\n    val_epoch_time = end_val_time - start_val_time\n    # Append validation metrics to lists\n    #val_losses.append(validation_loss)\n    #val_accuracies.append(validation_accuracy)\n    # Log Validation accuracy and loss to WandB\n    wandb.log({\"validation_loss\": validation_loss, \"validation_accuracy\": validation_accuracy})\n    print(f'Validation Loss: {validation_loss}, Validation Accuracy: {validation_accuracy}, Validation Time: {val_epoch_time} seconds')","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:18:02.372330Z","iopub.execute_input":"2024-01-23T09:18:02.372678Z","iopub.status.idle":"2024-01-23T09:48:03.954723Z","shell.execute_reply.started":"2024-01-23T09:18:02.372644Z","shell.execute_reply":"2024-01-23T09:48:03.953754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_name =  [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\nlabels_list = {i: labels_name[i] for i in range(10)}\nlabels_list","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:48:03.956314Z","iopub.execute_input":"2024-01-23T09:48:03.957096Z","iopub.status.idle":"2024-01-23T09:48:04.393402Z","shell.execute_reply.started":"2024-01-23T09:48:03.957065Z","shell.execute_reply":"2024-01-23T09:48:04.392327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Test Run and Submission File Generation","metadata":{}},{"cell_type":"code","source":"directory = '/kaggle/working/test'\nImageId = sorted([(''.join(filter(str.isdigit, name ))) for name in os.listdir(directory)])\n\nwith torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_iter):\n        output = model(data[0].to(device)).argmax(1)\n        predicted = output.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n    out_df = pd.DataFrame(np.c_[ImageId, test_pred.numpy()], columns=['id', 'label']).astype({'id':'int', 'label':'int'})\n    out_df['label'] = out_df['label'].map(labels_list)\nout_df.to_csv('ORsubmission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T09:48:04.394661Z","iopub.execute_input":"2024-01-23T09:48:04.395035Z","iopub.status.idle":"2024-01-23T09:50:58.626451Z","shell.execute_reply.started":"2024-01-23T09:48:04.394989Z","shell.execute_reply":"2024-01-23T09:50:58.625463Z"},"trusted":true},"execution_count":null,"outputs":[]}]}