{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3649,"databundleVersionId":46718,"sourceType":"competition"}],"dockerImageVersionId":30635,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"WanDB: https://wandb.ai/gagangt/cifar-10\n\nKaggle: https://www.kaggle.com/gaganreddygt/cifar-10/edit\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n        ","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:51:06.007759Z","iopub.execute_input":"2024-01-22T23:51:06.00866Z","iopub.status.idle":"2024-01-22T23:51:06.962361Z","shell.execute_reply.started":"2024-01-22T23:51:06.008615Z","shell.execute_reply":"2024-01-22T23:51:06.961408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Importing all the necessory libs","metadata":{}},{"cell_type":"code","source":"import glob\nfrom PIL import Image\nimport matplotlib.pyplot as plt\nimport collections\nimport math\nimport os\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import DataLoader, Dataset\nimport torchvision\nfrom torchvision import datasets, transforms\nfrom torchvision.models import resnet18\nnp.random.seed(0)\ntorch.manual_seed(0)\ntorch.cuda.manual_seed(0)\nimport wandb\nimport tqdm\nfrom tqdm import tqdm as tqdm_module\n# Log in to wandb with your project and experiment name\nwandb.init(project=\"cifar-10\", mode=disabled)","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:51:06.964524Z","iopub.execute_input":"2024-01-22T23:51:06.965306Z","iopub.status.idle":"2024-01-22T23:52:16.790794Z","shell.execute_reply.started":"2024-01-22T23:51:06.965268Z","shell.execute_reply":"2024-01-22T23:52:16.789566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install py7zr","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:52:16.792561Z","iopub.execute_input":"2024-01-22T23:52:16.793162Z","iopub.status.idle":"2024-01-22T23:52:33.240748Z","shell.execute_reply.started":"2024-01-22T23:52:16.793128Z","shell.execute_reply":"2024-01-22T23:52:33.239761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unziping the test and train dataset !! It really takes time :(","metadata":{}},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/train.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:52:33.242722Z","iopub.execute_input":"2024-01-22T23:52:33.243082Z","iopub.status.idle":"2024-01-22T23:53:31.282654Z","shell.execute_reply.started":"2024-01-22T23:52:33.243052Z","shell.execute_reply":"2024-01-22T23:53:31.281347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python -m py7zr x /kaggle/input/cifar-10/test.7z","metadata":{"execution":{"iopub.status.busy":"2024-01-22T23:53:31.286952Z","iopub.execute_input":"2024-01-22T23:53:31.288458Z","iopub.status.idle":"2024-01-23T00:12:55.706589Z","shell.execute_reply.started":"2024-01-22T23:53:31.288407Z","shell.execute_reply":"2024-01-23T00:12:55.705255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Reorganizing the dataset, creating multiple files within train_valid_test -> test, valid,train, train_valid. Except test set, all of them have segregated classe files.","metadata":{}},{"cell_type":"code","source":"data_dir = '/kaggle/working/'\n\ndef read_csv_labels(fname):\n    \"\"\"Read `fname` to return a filename to label dictionary.\"\"\"\n    with open(fname, 'r') as f:\n        # Skip the file header line (column name)\n        lines = f.readlines()[1:]\n    tokens = [l.rstrip().split(',') for l in lines]\n    return dict(((name, label) for name, label in tokens))\n\nlabels = read_csv_labels(os.path.join(data_dir, '/kaggle/input/cifar-10/trainLabels.csv'))\nprint(f'Number training examples: {len(labels)}')\nprint(f'Number classes: {len(set(labels.values()))}')\n\n\ndef copyfile(filename, target_dir):\n    \"\"\"Copy a file into a target directory.\"\"\"\n    os.makedirs(target_dir, exist_ok=True)\n    shutil.copy(filename, target_dir)\n\ndef reorg_train_valid(data_dir, labels, valid_ratio):\n    \"\"\"Split the validation set out of the original training set.\"\"\"\n    # The number of examples of the class that has the fewest examples in the\n    # training dataset\n    n = collections.Counter(labels.values()).most_common()[-1][1]\n    # The number of examples per class for the validation set\n    n_valid_per_label = max(1, math.floor(n * valid_ratio))\n    label_count = {}\n    for train_file in os.listdir(os.path.join(data_dir, 'train')):\n        label = labels[train_file.split('.')[0]]\n        fname = os.path.join(data_dir, 'train', train_file)\n        copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                     'train_valid', label))\n        if label not in label_count or label_count[label] < n_valid_per_label:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'valid', label))\n            label_count[label] = label_count.get(label, 0) + 1\n        else:\n            copyfile(fname, os.path.join(data_dir, 'train_valid_test',\n                                         'train', label))\n    return n_valid_per_label\n\ndef reorg_test(data_dir):\n    \"\"\"Organize the testing set for data loading during prediction.\"\"\"\n    for test_file in os.listdir(os.path.join(data_dir, 'test')):\n        copyfile(os.path.join(data_dir, 'test', test_file),\n                 os.path.join(data_dir, 'train_valid_test', 'test',\n                              'unknown'))\n        \ndef reorg_cifar10_data(data_dir, valid_ratio):\n    labels = read_csv_labels('/kaggle/input/cifar-10/trainLabels.csv')\n    reorg_train_valid(data_dir, labels, valid_ratio)\n    \n    label_mapping = {'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship':8, 'truck':9}\n    labels = {key: label_mapping[value] for key, value in labels.items()}\n    \n    reorg_test(data_dir)\n\nbatch_size = 64\nvalid_ratio = 0.1\nreorg_cifar10_data(data_dir, valid_ratio)","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:12:55.708493Z","iopub.execute_input":"2024-01-23T00:12:55.708926Z","iopub.status.idle":"2024-01-23T00:13:56.277689Z","shell.execute_reply.started":"2024-01-23T00:12:55.708875Z","shell.execute_reply":"2024-01-23T00:13:56.276587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add multple Transformations. Too many was not too good, it dropped the accuracy to 20 :(    so commented a few one by one.","metadata":{}},{"cell_type":"code","source":"#TRANSFORMATIONS\n\n# Basic transformations\nbase_transform = transforms.Compose([\n    transforms.ToTensor(),\n    transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]),\n])\n\n# Additional augmentations\nadditional_transformations = transforms.Compose([\n    torchvision.transforms.RandomResizedCrop(224),\n    transforms.RandomHorizontalFlip(),\n    #transforms.RandomRotation(degrees=15),\n    #transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.2),\n    #transforms.RandomResizedCrop(size=(32, 32), scale=(0.8, 1.0), ratio=(0.75, 1.333)),\n    #transforms.RandomAffine(degrees=0, translate=(0.1, 0.1), scale=(0.9, 1.1), shear=0),\n])\n\n# Combine both for training\ntransform_train = transforms.Compose([\n    base_transform,\n    additional_transformations,\n])\n\n# Keep testing transformations the same\ntransform_test = transforms.Compose({\n    torchvision.transforms.Resize(256),\n    base_transform\n})\n\ntrain_ds, train_valid_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_train) for folder in ['train', 'train_valid']]\n\nvalid_ds, test_ds = [torchvision.datasets.ImageFolder(\n    os.path.join(data_dir, 'train_valid_test', folder),\n    transform=transform_test) for folder in ['valid', 'test']]\n\ntrain_iter, train_valid_iter = [torch.utils.data.DataLoader(\n    dataset, batch_size, shuffle=True, drop_last=True)\n    for dataset in (train_ds, train_valid_ds)]\n\nvalid_iter = torch.utils.data.DataLoader(valid_ds, batch_size, shuffle=False,\n                                         drop_last=True)\n\ntest_iter = torch.utils.data.DataLoader(test_ds, batch_size, shuffle=False,\n                                        drop_last=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:13:56.278897Z","iopub.execute_input":"2024-01-23T00:13:56.282263Z","iopub.status.idle":"2024-01-23T00:13:58.390817Z","shell.execute_reply.started":"2024-01-23T00:13:56.282228Z","shell.execute_reply":"2024-01-23T00:13:58.389758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  MODEL","metadata":{}},{"cell_type":"markdown","source":"Model. Dropout and the SGD with lr scheduler works really good.","metadata":{}},{"cell_type":"code","source":"from torch.optim.lr_scheduler import StepLR\n\n# Load pre-trained ResNet-18 model\nmodel = torchvision.models.resnet18(pretrained=True)\n\n# Modify the final fully connected layer with dropout\nmodel.fc = nn.Sequential(\n    nn.Dropout(0.5),\n    nn.Linear(model.fc.in_features, 10)\n)\n\nprint(\"CUDA available:\", torch.cuda.is_available())\nif torch.cuda.is_available():\n    print(\"Current GPU device:\", torch.cuda.current_device())\n    print(\"GPU device name:\", torch.cuda.get_device_name(torch.cuda.current_device()))\n\n# Move the model to the device (CPU or GPU)\ndevice = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\nmodel.to(device)\n\n#Define loss function and optimizer\ncriterion = nn.CrossEntropyLoss()\noptimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)\n\n # Learning rate schedule\nscheduler = StepLR(optimizer, step_size=5, gamma=0.1)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:13:58.392164Z","iopub.execute_input":"2024-01-23T00:13:58.392557Z","iopub.status.idle":"2024-01-23T00:13:59.290268Z","shell.execute_reply.started":"2024-01-23T00:13:58.392521Z","shell.execute_reply":"2024-01-23T00:13:59.28915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Lets train and evaluate the Model \n","metadata":{}},{"cell_type":"code","source":"import logging\nfrom tqdm import tqdm\nimport os\nimport torch\nimport numpy as np\nfrom datetime import datetime\nimport wandb\n\n\n# Set up logging\nlog_file = 'training_log.txt'\nlogging.basicConfig(filename=log_file, level=logging.INFO, format='%(asctime)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S')\n\n# Define early stopping and checkpoint parameters\nearly_stop_patience = 5\nearly_stop_counter = 0\nbest_validation_loss = np.inf\n\n# Training the model\nnum_epochs = 10\n\n# Inside the training loop:\nfor epoch in range(num_epochs):\n    model.train()\n    running_loss = 0.0\n    correct_train = 0\n    total_train = 0\n\n    progress_bar = tqdm(train_iter, desc=f'Epoch {epoch+1}/{num_epochs}', leave=False)\n\n    for inputs, labels in progress_bar:\n        inputs, labels = inputs.to(device), labels.to(device)\n        optimizer.zero_grad()\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        running_loss += loss.item()\n\n        _, predicted_train = torch.max(outputs.data, 1)\n        total_train += labels.size(0)\n        correct_train += (predicted_train == labels).sum().item()\n\n    # Log training loss and accuracy to wandb\n    wandb.log({\"Training Loss\": running_loss/len(train_iter),\n               \"Training Accuracy\": 100 * correct_train / total_train})\n\n    # Evaluating the model on the validation set\n    model.eval()\n    correct_valid = 0\n    total_valid = 0\n    validation_loss = 0.0\n\n    with torch.no_grad():\n        for inputs, labels in valid_iter:\n            inputs, labels = inputs.to(device), labels.to(device)\n            outputs = model(inputs)\n            loss = criterion(outputs, labels)\n            validation_loss += loss.item()\n\n            _, predicted_valid = torch.max(outputs.data, 1)\n            total_valid += labels.size(0)\n            correct_valid += (predicted_valid == labels).sum().item()\n\n    # Compute average validation loss\n    avg_validation_loss = validation_loss / len(valid_iter)\n\n    # Log average validation loss and accuracy to wandb\n    wandb.log({\"Validation Loss\": avg_validation_loss,\n               \"Validation Accuracy\": 100 * correct_valid / total_valid})\n\n    # Update early stopping criteria\n    if avg_validation_loss < best_validation_loss:\n        best_validation_loss = avg_validation_loss\n        early_stop_counter = 0\n\n        # Save the model checkpoint with the epoch number\n        timestamp = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n        checkpoint_path = f'checkpoint_epoch_{epoch + 1}_{timestamp}.pth'\n        torch.save(model.state_dict(), checkpoint_path)\n\n        # Log checkpoint information to wandb\n        wandb.log({\"Best Validation Loss\": best_validation_loss, \"Checkpoint Path\": checkpoint_path})\n        logging.info(f'Epoch {epoch + 1}: Checkpoint saved at {checkpoint_path}')\n    else:\n        early_stop_counter += 1\n\n    # Print metrics after each epoch\n    print(f\"Epoch {epoch+1}/{num_epochs}, \"\n          f\"Training Loss: {running_loss/len(train_iter)}, \"\n          f\"Training Accuracy: {100 * correct_train / total_train}%, \"\n          f\"Validation Loss: {avg_validation_loss}, \"\n          f\"Validation Accuracy: {100 * correct_valid / total_valid}%\")\n\n    # Early stopping check\n    if early_stop_counter >= early_stop_patience:\n        print(f\"Early stopping after {early_stop_patience} epochs of increasing validation loss.\")\n        break  # Break out of the training loop\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:13:59.291716Z","iopub.execute_input":"2024-01-23T00:13:59.292077Z","iopub.status.idle":"2024-01-23T00:38:26.449Z","shell.execute_reply.started":"2024-01-23T00:13:59.292043Z","shell.execute_reply":"2024-01-23T00:38:26.447823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Running a test Loop on the train_valid-ds","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')  \n\n# Create a DataLoader for the train_valid_ds\ntrain_valid_loader = DataLoader(train_valid_ds, batch_size, shuffle=False)\n\n# Set your model to evaluation mode\nmodel.eval()\n\n# Initialize variables to store loss and predictions\ntotal_loss = 0.0\nall_predictions = []\nall_labels = []\n\n# Disable gradient computation during testing\nwith torch.no_grad():\n    for batch in train_valid_loader:\n        inputs, labels = batch\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        # Forward pass\n        outputs = model(inputs)\n        loss = criterion(outputs, labels)\n\n        # Record loss\n        total_loss += loss.item()\n\n        # Record predictions and labels\n        _, predictions = torch.max(outputs, 1)\n        all_predictions.extend(predictions.cpu().numpy())\n        all_labels.extend(labels.cpu().numpy())\n\n# Calculate average loss\naverage_Test_loss_train_valid = total_loss / len(train_valid_loader)\n\n# Calculate accuracy\naccuracy_Test_train_valid = accuracy_score(all_labels, all_predictions)\nwandb.log({\"Train valid ds loss (test)\": best_validation_loss, \"Train vaild ds Accuracy\": accuracy_Test_train_valid})\nprint(f'Average Loss: {average_Test_loss_train_valid:.4f}, Accuracy: {accuracy_Test_train_valid * 100:.2f}%')\n\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:43:52.136135Z","iopub.execute_input":"2024-01-23T00:43:52.13657Z","iopub.status.idle":"2024-01-23T00:45:29.488745Z","shell.execute_reply.started":"2024-01-23T00:43:52.136537Z","shell.execute_reply":"2024-01-23T00:45:29.487649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Final Test on the test ds and saving the pred into submission.csv ","metadata":{}},{"cell_type":"code","source":"labels_name = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\", \"frog\", \"horse\", \"ship\", \"truck\"]\nlabels_list = {str(i): labels_name[i] for i in range(10)}\n\ndirectory = '/kaggle/working/test'\nImageId = sorted([(''.join(filter(str.isdigit, name))) for name in os.listdir(directory)])\n\nwith torch.no_grad():\n    model.eval()\n    test_pred = torch.LongTensor()\n    for i, data in enumerate(test_iter):\n        output = model(data[0].to(device)).argmax(1)\n        predicted = output.cpu()\n        test_pred = torch.cat((test_pred, predicted), dim=0)\n\n# Create the DataFrame\nout_df = pd.DataFrame(np.c_[ImageId, test_pred.numpy()], columns=['id', 'label']).astype({'id': 'int', 'label': 'str'})\nout_df['label'] = out_df['label'].map(labels_list)\n\n# Sort the DataFrame by 'id' as strings\nout_df['id'] = out_df['id'].astype(str)\nout_df = out_df.sort_values(by='id')\n\nout_df.to_csv('Submission.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2024-01-23T00:55:47.506437Z","iopub.execute_input":"2024-01-23T00:55:47.50682Z","iopub.status.idle":"2024-01-23T01:05:19.981987Z","shell.execute_reply.started":"2024-01-23T00:55:47.506788Z","shell.execute_reply":"2024-01-23T01:05:19.980824Z"},"trusted":true},"execution_count":null,"outputs":[]}]}