{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":87323,"databundleVersionId":9934427,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Step 1: Installing necessary libraries","metadata":{}},{"cell_type":"code","source":"# !pip install torch torchvision pandas scikit-learn matplotlib tqdm torchmetrics","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-11T08:08:46.892037Z","iopub.execute_input":"2024-11-11T08:08:46.892517Z","iopub.status.idle":"2024-11-11T08:08:46.898463Z","shell.execute_reply.started":"2024-11-11T08:08:46.892464Z","shell.execute_reply":"2024-11-11T08:08:46.897190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"GPU Check","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:46.900180Z","iopub.execute_input":"2024-11-11T08:08:46.900574Z","iopub.status.idle":"2024-11-11T08:08:48.049492Z","shell.execute_reply.started":"2024-11-11T08:08:46.900535Z","shell.execute_reply":"2024-11-11T08:08:48.048257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 2: Importing required libraries","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torchvision.transforms as transforms\nfrom torchvision.transforms.v2 import GaussianNoise\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset, random_split, Subset\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom torchmetrics.classification import BinaryAccuracy, BinaryF1Score, BinaryAUROC\nimport pandas as pd\nfrom PIL import Image\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os\nfrom tqdm import tqdm\nimport torch.cuda.amp as amp\nfrom glob import glob\nfrom transformers import get_cosine_schedule_with_warmup\nfrom transformers import AutoImageProcessor, BeitForImageClassification, AutoModelForImageClassification, SwinForImageClassification, ViTForImageClassification, ConvNextForImageClassification\nimport shutil","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:48.051671Z","iopub.execute_input":"2024-11-11T08:08:48.052048Z","iopub.status.idle":"2024-11-11T08:08:48.060999Z","shell.execute_reply.started":"2024-11-11T08:08:48.052007Z","shell.execute_reply":"2024-11-11T08:08:48.059864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 3: Custom Dataset Class\n\nThis dataset class will read the image paths and labels from a CSV file, handle basic image augmentations, and load images efficiently.","metadata":{}},{"cell_type":"code","source":"class CustomImageDataset(Dataset):\n    def __init__(self, csv_file, img_dir, transform=None):\n        self.annotations = csv_file\n        self.img_dir     = img_dir #glob(os.path.join(img_dir, \"*.jpg\"))\n        self.transform   = transform\n        assert len(self.img_dir) == len(self.img_dir)\n\n    def __len__(self):\n        return len(self.annotations)\n\n    def __getitem__(self, idx):\n        img_path = os.path.join(self.img_dir, self.annotations.iloc[idx, 0])\n        image    = Image.open(img_path).convert(\"RGB\")\n        label    = torch.tensor(int(self.annotations.iloc[idx, 1]))\n\n        if self.transform:\n            image = self.transform(image)\n            \n        return image, label","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:48.062232Z","iopub.execute_input":"2024-11-11T08:08:48.062531Z","iopub.status.idle":"2024-11-11T08:08:48.073085Z","shell.execute_reply.started":"2024-11-11T08:08:48.062500Z","shell.execute_reply":"2024-11-11T08:08:48.072159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 4: Creating Data Augmentation and DataLoader\n\nWe’ll use basic augmentation (horizontal flip), and split the dataset into training and validation sets.","metadata":{}},{"cell_type":"code","source":"def create_dataloaders(csv_file, img_dir, img_size=(224, 224), batch_size=32, n_fold=0):\n    # Define transforms with basic augmentations\n    transform = transforms.Compose([\n        transforms.Resize(img_size),\n        transforms.ToTensor(), \n        transforms.GaussianBlur(7), \n        transforms.RandomHorizontalFlip(), \n        transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2)\n    ])\n\n    # Initialize dataset\n    dataset = CustomImageDataset(csv_file=csv_file, img_dir=img_dir, transform=transform)\n    \n    # Create train/validation split\n    skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2024)\n    for i, (train_index, val_index) in enumerate(skf.split(np.zeros(len(csv_file)), csv_file.iloc[:, 1].values)):\n        if i == n_fold:\n            break\n            \n    train_dataset = Subset(dataset, train_index)\n    dataset = CustomImageDataset(csv_file=csv_file, img_dir=img_dir, \n                                 transform=transforms.Compose([transforms.Resize(img_size), \n                                                              transforms.ToTensor()]))\n    val_dataset = Subset(dataset, val_index)\n    print(len(train_dataset))\n    print(len(val_dataset))\n\n    # Create DataLoaders\n    train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=4, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n    \n    return train_loader, val_loader","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:33:53.953952Z","iopub.execute_input":"2024-11-11T08:33:53.954636Z","iopub.status.idle":"2024-11-11T08:33:53.965207Z","shell.execute_reply.started":"2024-11-11T08:33:53.954589Z","shell.execute_reply":"2024-11-11T08:33:53.964115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 5: Model Defition","metadata":{}},{"cell_type":"markdown","source":"Step 6: Training and Validation Functions\n\nMixed precision training using PyTorch’s AMP (Automatic Mixed Precision) is used to speed up training.","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, train_loader, criterion, optimizer, scheduler, device):\n    model.train()\n    running_loss = 0.0\n    for images, labels in tqdm(train_loader):\n        images, labels = images.to(device), labels.to(device).float().unsqueeze(1)\n\n        optimizer.zero_grad()\n        \n        outputs = model(images).logits[:, :1]\n        loss = criterion(outputs, labels)\n\n        # Backward pass and optimization step\n        loss.backward()\n        optimizer.step()\n        scheduler.step()\n\n        running_loss += loss.item() * images.size(0)\n\n    epoch_loss = running_loss / len(train_loader.dataset)\n    return epoch_loss\n\n\ndef validate(model, val_loader, criterion, device):\n    model.eval()\n    running_loss = 0.0\n    all_labels = []\n    all_outputs = []\n    \n    with torch.no_grad():\n        for images, labels in tqdm(val_loader):\n            images, labels = images.to(device), labels.to(device).float().unsqueeze(1)\n\n            outputs = model(images).logits[:, :1]\n            loss = criterion(outputs, labels)\n\n            running_loss += loss.item() * images.size(0)\n            all_labels.append(labels.cpu().numpy())\n            all_outputs.append(outputs.cpu().numpy())\n\n    epoch_loss = running_loss / len(val_loader.dataset)\n    \n    all_labels = np.concatenate(all_labels)\n    all_outputs = np.concatenate(all_outputs)\n    all_outputs = torch.sigmoid(torch.tensor(all_outputs)).numpy()  # Convert logits to probabilities\n    \n    return epoch_loss, all_labels, all_outputs","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:48.089116Z","iopub.execute_input":"2024-11-11T08:08:48.089492Z","iopub.status.idle":"2024-11-11T08:08:48.101815Z","shell.execute_reply.started":"2024-11-11T08:08:48.089459Z","shell.execute_reply":"2024-11-11T08:08:48.100815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EarlyStopping:\n    def __init__(self, patience=7, verbose=False, delta=0):\n        self.patience = patience\n        self.verbose = verbose\n        self.counter = 0\n        self.best_score = None\n        self.early_stop = False\n        self.val_loss_min = np.Inf\n        self.delta = delta\n\n    def __call__(self, val_loss, model, path):\n        score = -val_loss\n        if self.best_score is None:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n        elif score < self.best_score + self.delta:\n            self.counter += 1\n            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')\n            if self.counter >= self.patience:\n                self.early_stop = True\n        else:\n            self.best_score = score\n            self.save_checkpoint(val_loss, model, path)\n            self.counter = 0\n\n    def save_checkpoint(self, val_loss, model, path):\n        if self.verbose:\n            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')\n        torch.save(model.state_dict(), path + '/' + 'checkpoint.pth')\n        self.val_loss_min = val_loss","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:48.103065Z","iopub.execute_input":"2024-11-11T08:08:48.103860Z","iopub.status.idle":"2024-11-11T08:08:48.116072Z","shell.execute_reply.started":"2024-11-11T08:08:48.103819Z","shell.execute_reply":"2024-11-11T08:08:48.115163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 7: Training Loop and Metrics Calculation","metadata":{}},{"cell_type":"code","source":"def train_model(csv_file, img_dir, model, model_name, img_size=(224, 224), num_epochs=10, batch_size=32, lr=1e-4, n_fold=0, device='cuda', patience=3, warmup_epochs=0):\n    train_loader, val_loader = create_dataloaders(csv_file, img_dir, img_size=img_size, batch_size=batch_size, n_fold=n_fold)\n    train_sets = len(train_loader)\n\n    model     = model.to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = optim.Adam(model.parameters(), lr=lr)\n    scheduler = get_cosine_schedule_with_warmup(optimizer, train_sets * warmup_epochs, train_sets * num_epochs)\n    early_stopping = EarlyStopping(patience=patience, verbose=True)\n\n    train_losses = []\n    val_losses   = []\n    \n    path = model_name + str(n_fold)\n    os.makedirs(path, exist_ok=True)\n    \n    for epoch in range(num_epochs):\n        print(f'Epoch {epoch+1}/{num_epochs}')\n        train_loss = train_one_epoch(model, train_loader, criterion, optimizer, scheduler, device)\n        val_loss, val_labels, val_outputs = validate(model, val_loader, criterion, device)\n\n        train_losses.append(train_loss)\n        val_losses.append(val_loss)\n        \n        # Calculate metrics on validation set\n        val_preds = (val_outputs > 0.5).astype(int)\n        accuracy = accuracy_score(val_labels, val_preds)\n        roc_auc = roc_auc_score(val_labels, val_outputs)\n        print(f'Train Loss: {train_loss:.4f}, Val Loss: {val_loss:.4f}, Val Accuracy: {accuracy:.4f}, Val ROC AUC : {roc_auc:.4f}')\n        early_stopping(val_loss, model, path)\n        if early_stopping.early_stop:\n            print(\"Early stopping\")\n            break\n        print('Updating learning rate to {}'.format(scheduler.get_last_lr()[0]))\n        \n    # Plot Loss\n    plt.plot(train_losses, label='Train Loss')\n    plt.plot(val_losses, label='Validation Loss')\n    plt.legend()\n    plt.savefig('loss_plot.png')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:08:48.117416Z","iopub.execute_input":"2024-11-11T08:08:48.118088Z","iopub.status.idle":"2024-11-11T08:08:48.130172Z","shell.execute_reply.started":"2024-11-11T08:08:48.118054Z","shell.execute_reply":"2024-11-11T08:08:48.129416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Step 8: Running the Training\n\nYou can run the training with the following command:","metadata":{}},{"cell_type":"code","source":"labels = pd.read_csv (\"/kaggle/input/cidaut-ai-fake-scene-classification-2024/train.csv\")\nlabels[\"label\"] = labels[\"label\"].map({\"editada\":0, \"real\":1})\nimg_dir = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Train\"\nmodel = BeitForImageClassification.from_pretrained('microsoft/beit-base-patch16-384')\nbatch_size = 16\nlr = 1e-6\nimg_size = (384, 384)\nn_fold = 0\ntrain_model(labels, img_dir, model, 'microsoft/beit-base-patch16-384', img_size=img_size, num_epochs=30, \n            batch_size=batch_size, lr=lr, n_fold=n_fold, patience=3, warmup_epochs=0)\ntorch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:34:33.995622Z","iopub.execute_input":"2024-11-11T08:34:33.996324Z","iopub.status.idle":"2024-11-11T08:54:59.139895Z","shell.execute_reply.started":"2024-11-11T08:34:33.996284Z","shell.execute_reply":"2024-11-11T08:54:59.138752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict(csv_file, img_dir, model, model_name, img_size=(224, 224), batch_size=32, n_fold=0, device='cuda', delete=False):\n    model     = model.to(device) # load the model into the GPU\n    model.load_state_dict(torch.load(os.path.join(model_name + str(n_fold), 'checkpoint.pth')))\n    criterion = nn.BCEWithLogitsLoss()\n    \n    transform = transforms.Compose([\n        transforms.Resize(img_size),\n        transforms.ToTensor()\n    ])\n    test_dataset = CustomImageDataset(csv_file=csv_file, img_dir=img_dir, transform=transform)\n    test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4, pin_memory=True)\n    _, _, outputs = validate(model, test_loader, criterion, device)\n    if delete:\n        shutil.rmtree(model_name + str(n_fold))\n    return outputs","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:54:59.141991Z","iopub.execute_input":"2024-11-11T08:54:59.142327Z","iopub.status.idle":"2024-11-11T08:54:59.150261Z","shell.execute_reply.started":"2024-11-11T08:54:59.142287Z","shell.execute_reply":"2024-11-11T08:54:59.149415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = pd.read_csv (\"/kaggle/input/cidaut-ai-fake-scene-classification-2024/sample_submission.csv\")\nimg_dir = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Test\"\npreds = predict(labels, img_dir, model, 'microsoft/beit-base-patch16-384', img_size=img_size, batch_size=batch_size, device='cuda', n_fold=n_fold)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:54:59.151398Z","iopub.execute_input":"2024-11-11T08:54:59.151684Z","iopub.status.idle":"2024-11-11T08:55:04.143700Z","shell.execute_reply.started":"2024-11-11T08:54:59.151653Z","shell.execute_reply":"2024-11-11T08:55:04.142364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels['label'] = preds\nlabels.head()","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:55:08.118425Z","iopub.execute_input":"2024-11-11T08:55:08.118870Z","iopub.status.idle":"2024-11-11T08:55:08.132322Z","shell.execute_reply.started":"2024-11-11T08:55:08.118817Z","shell.execute_reply":"2024-11-11T08:55:08.131225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-11T08:55:12.429576Z","iopub.execute_input":"2024-11-11T08:55:12.429975Z","iopub.status.idle":"2024-11-11T08:55:12.439001Z","shell.execute_reply.started":"2024-11-11T08:55:12.429937Z","shell.execute_reply":"2024-11-11T08:55:12.438163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TIPS\n- check AUC-ROC for validation\n- improve the model `SimpleCNN`\n- inspect the data\n- image augmentations\n- do the code for predicting the images on `Test/` based on the `validate` function\n- submit the `sample_submission.csv` and check the format\n","metadata":{}}]}