{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":87323,"databundleVersionId":9934427,"sourceType":"competition"},{"sourceId":10381877,"sourceType":"datasetVersion","datasetId":6431294}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Vision Transformers are excellent for tasks involving high-dimensional data, such as image classification, especially when large datasets are available.\n\nStrengths:\n\n    Handles global context well, which is essential for subtle differences.\n        \n    Can be fine-tuned on small datasets effectively.","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\nfrom torchvision import transforms\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import ViTForImageClassification, ViTFeatureExtractor\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom PIL import Image","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:42:45.494158Z","iopub.execute_input":"2025-01-06T16:42:45.494384Z","iopub.status.idle":"2025-01-06T16:43:02.229434Z","shell.execute_reply.started":"2025-01-06T16:42:45.494362Z","shell.execute_reply":"2025-01-06T16:43:02.228749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class CustomDataset(Dataset):\n    def __init__(self, image_paths, labels=None, transform=None):\n        \"\"\"\n        Args:\n            image_paths (list): List of image file paths.\n            labels (list, optional): List of labels corresponding to the images.\n            transform (callable, optional): Transform to be applied to the images.\n        \"\"\"\n        self.image_paths = image_paths\n        self.labels = labels\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_paths)\n\n    def __getitem__(self, idx):\n        img_path = self.image_paths[idx]\n        image = Image.open(img_path).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n\n        if self.labels is not None:\n            label = self.labels[idx]\n            return image, torch.tensor(label, dtype=torch.float32)\n        else:\n            return image, os.path.basename(img_path)  # For test set","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T17:01:08.189304Z","iopub.execute_input":"2025-01-06T17:01:08.189669Z","iopub.status.idle":"2025-01-06T17:01:08.195022Z","shell.execute_reply.started":"2025-01-06T17:01:08.189638Z","shell.execute_reply":"2025-01-06T17:01:08.194197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_vit_model(train_loader, val_loader, device, epochs=10, lr=5e-5):\n    \"\"\"Trains the ViT model.\"\"\"\n    # Load pretrained ViT model\n    model = ViTForImageClassification.from_pretrained(\n        \"google/vit-base-patch16-224-in21k\",\n        num_labels=1,\n        problem_type=\"binary_classification\"\n    )\n    model.to(device)\n\n    # Loss and optimizer\n    criterion = nn.BCEWithLogitsLoss()\n    optimizer = optim.AdamW(model.parameters(), lr=lr)\n\n    best_auc = 0\n    for epoch in range(epochs):\n        # Training\n        model.train()\n        train_loss = 0\n        for images, labels in train_loader:\n            images, labels = images.to(device), labels.to(device)\n            optimizer.zero_grad()\n            outputs = model(images).logits.squeeze()\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            train_loss += loss.item()\n\n        # Validation\n        model.eval()\n        val_loss = 0\n        all_labels = []\n        all_preds = []\n        with torch.no_grad():\n            for images, labels in val_loader:\n                images, labels = images.to(device), labels.to(device)\n                outputs = model(images).logits.squeeze()\n                loss = criterion(outputs, labels)\n                val_loss += loss.item()\n                all_labels.extend(labels.cpu().numpy())\n                all_preds.extend(torch.sigmoid(outputs).cpu().numpy())\n\n        val_auc = roc_auc_score(all_labels, all_preds)\n        print(f\"Epoch {epoch + 1}/{epochs} | Train Loss: {train_loss / len(train_loader):.4f} | \"\n              f\"Val Loss: {val_loss / len(val_loader):.4f} | Val AUC: {val_auc:.4f}\")\n\n        # Save best model\n        if val_auc > best_auc:\n            best_auc = val_auc\n            torch.save(model.state_dict(), \"best_vit_model.pth\")\n\n    return model\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T17:01:09.375772Z","iopub.execute_input":"2025-01-06T17:01:09.376082Z","iopub.status.idle":"2025-01-06T17:01:09.383105Z","shell.execute_reply.started":"2025-01-06T17:01:09.376053Z","shell.execute_reply":"2025-01-06T17:01:09.382423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_with_vit(model, test_loader, device, submission_path):\n    \"\"\"Generates predictions using the trained ViT model.\"\"\"\n    model.eval()\n    predictions = []\n    image_names = []\n    with torch.no_grad():\n        for images, names in test_loader:\n            images = images.to(device)\n            outputs = model(images).logits.squeeze()\n            predictions.extend(torch.sigmoid(outputs).cpu().numpy())\n            image_names.extend(names)\n\n    # Save submission file\n    submission_df = pd.DataFrame({\n        \"image\": image_names,\n        \"label\": predictions\n    })\n    submission_df.to_csv(submission_path, index=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T16:43:23.557258Z","iopub.execute_input":"2025-01-06T16:43:23.557593Z","iopub.status.idle":"2025-01-06T16:43:23.563302Z","shell.execute_reply.started":"2025-01-06T16:43:23.557566Z","shell.execute_reply":"2025-01-06T16:43:23.562434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if __name__ == \"__main__\":\n    # Configuration\n    TRAIN_DIR = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Train\"\n    TEST_DIR = \"/kaggle/input/cidaut-ai-fake-scene-classification-2024/Test\"\n    LABELS_PATH = \"/kaggle/input/ttaaaa/train.csv\"  \n    BATCH_SIZE = 16\n    EPOCHS = 10\n    IMG_SIZE = 224\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n    # Read labels\n    df_labels = pd.read_csv(LABELS_PATH)\n    image_paths = [os.path.join(TRAIN_DIR, img) for img in df_labels[\"image_name\"]]\n    labels = df_labels[\"label\"].astype(float).values\n\n    # Split data\n    train_paths, val_paths, train_labels, val_labels = train_test_split(\n        image_paths, labels, test_size=0.2, random_state=42\n    )\n\n    # Transformations\n    train_transform = transforms.Compose([\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.RandomHorizontalFlip(),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n    ])\n    val_transform = transforms.Compose([\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n        transforms.ToTensor(),\n        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])\n    ])\n\n    # Create datasets and loaders\n    train_dataset = CustomDataset(train_paths, train_labels, transform=train_transform)\n    val_dataset = CustomDataset(val_paths, val_labels, transform=val_transform)\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    # Train the model\n    vit_model = train_vit_model(train_loader, val_loader, DEVICE, epochs=EPOCHS)\n\n    # Prepare test dataset and loader\n    test_files = sorted([f for f in os.listdir(TEST_DIR) if f.endswith(\".jpg\")])\n    test_paths = [os.path.join(TEST_DIR, f) for f in test_files]\n    test_dataset = CustomDataset(test_paths, transform=val_transform)\n    test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    # Predict and save submission\n    predict_with_vit(vit_model, test_loader, DEVICE, \"submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-06T17:01:56.212252Z","iopub.execute_input":"2025-01-06T17:01:56.212580Z","iopub.status.idle":"2025-01-06T17:06:43.359432Z","shell.execute_reply.started":"2025-01-06T17:01:56.212551Z","shell.execute_reply":"2025-01-06T17:06:43.358698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}