{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":106372,"databundleVersionId":12864984,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install -q torch-ema\n\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim.lr_scheduler import OneCycleLR, ReduceLROnPlateau\nfrom torchvision import transforms, models\nfrom torch.utils.data import Dataset, DataLoader, WeightedRandomSampler\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, accuracy_score\nimport os\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nimport timm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport random\nfrom torch_ema import ExponentialMovingAverage\nfrom torch.cuda.amp import GradScaler, autocast\n\n\n\n# Set device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load and prepare data\ndf = pd.read_csv(\"/kaggle/input/2025-bamboo-summer-competiton-dl-pr/train.csv\")\ndf['image_path'] = \"/kaggle/input/2025-bamboo-summer-competiton-dl-pr/train/\" + df['filename']\n\n# Encode labels\nunique_labels = sorted(df['label'].unique())\nlabel2id = {label: idx for idx, label in enumerate(unique_labels)}\nid2label = {idx: label for label, idx in label2id.items()}\ndf['label_id'] = df['label'].map(label2id)\n\n# Split into train/valid\ntrain_df, valid_df = train_test_split(df, test_size=0.1, stratify=df['label_id'], random_state=42)\n\ntrain_transform = A.Compose([\n    A.RandomResizedCrop(size=(256, 256), scale=(0.7, 1.0), ratio=(0.8, 1.2), p=1.0),\n    A.HorizontalFlip(p=0.5),\n    A.VerticalFlip(p=0.3),\n    A.Affine(translate_percent=0.1, scale=(0.8, 1.2), rotate=(-30, 30), p=0.8),\n    A.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1, p=0.8),\n    A.CoarseDropout(p=0.5),\n    A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=30, val_shift_limit=20, p=0.8),\n    A.OpticalDistortion(p=0.3),\n    A.RandomFog(p=0.2),  # Optional\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\n\nvalid_transform = A.Compose([\n    A.Resize(height=256, width=256),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\n# Dataset class with Albumentations support\nclass ButterflyDataset(Dataset):\n    def __init__(self, dataframe, transform=None):\n        self.dataframe = dataframe\n        self.transform = transform\n        \n    def __len__(self):\n        return len(self.dataframe)\n    \n    def __getitem__(self, idx):\n        img_path = self.dataframe.iloc[idx]['image_path']\n        image = Image.open(img_path).convert('RGB')\n        label = self.dataframe.iloc[idx]['label_id']\n        \n        if self.transform:\n            image = self.transform(image=np.array(image))['image']\n            \n        return image, label\n\n# Create datasets\ntrain_dataset = ButterflyDataset(train_df, train_transform)\nvalid_dataset = ButterflyDataset(valid_df, valid_transform)\n\n# Class Balancing\nclass_counts = dict(sorted(Counter(train_df['label_id']).items()))\ntotal_samples = sum(class_counts.values())\nclass_weights = {cls: total_samples/(len(class_counts)*count) for cls, count in class_counts.items()}\nsample_weights = [class_weights[cls] for cls in train_df['label_id']]\nsampler = WeightedRandomSampler(sample_weights, len(sample_weights), replacement=True)\n\n# DataLoaders\ntrain_loader = DataLoader(train_dataset, batch_size=64, sampler=sampler, num_workers=4)\nvalid_loader = DataLoader(valid_dataset, batch_size=64, shuffle=False, num_workers=4)\n\n# Hybrid Loss combining Label Smoothing and Focal Loss\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nclass HybridLoss(nn.Module):\n    def __init__(self, num_classes, alpha=None, gamma=2.0, smoothing=0.1):\n        super().__init__()\n        self.alpha = alpha\n        self.gamma = gamma\n        self.smoothing = smoothing\n        self.num_classes = num_classes\n        \n    def forward(self, inputs, targets):\n        # Convert targets to one-hot encoding\n        targets_onehot = F.one_hot(targets, num_classes=self.num_classes).float()\n        \n        # Apply label smoothing\n        targets_onehot = targets_onehot * (1 - self.smoothing) + self.smoothing / self.num_classes\n        \n        log_probs = F.log_softmax(inputs, dim=-1)\n        \n        # Calculate focal loss\n        loss = -((1 - torch.exp(log_probs)) ** self.gamma * targets_onehot * log_probs)\n        \n        # Apply class weights if provided\n        if self.alpha is not None:\n            alpha_weight = self.alpha[targets].unsqueeze(1)\n            loss = alpha_weight * loss\n            \n        return loss.mean()\n\n# When initializing the loss function:\nnum_classes = len(label2id)  # Make sure this matches your model's output\nalpha = torch.tensor([class_weights[i] for i in sorted(class_weights.keys())]).float().to(device)\ncriterion = HybridLoss(num_classes=num_classes, alpha=alpha, gamma=2.0, smoothing=0.1)\n\n\nmodel = timm.create_model('convnext_base', pretrained=True)\nmodel.reset_classifier(len(label2id))\nmodel = model.to(device)\nema = ExponentialMovingAverage(model.parameters(), decay=0.999)\nscaler = GradScaler()\n\n# Optimizer configuration with differential learning rates\noptimizer = optim.AdamW([\n    {'params': model.stem.parameters(), 'lr': 1e-5},\n    {'params': model.stages.parameters(), 'lr': 3e-5},\n    {'params': model.head.parameters(), 'lr': 1e-4}\n], weight_decay=0.05)\n\n\n# OneCycleLR scheduler\nscheduler = OneCycleLR(optimizer, max_lr=1e-3, \n                      steps_per_epoch=len(train_loader), \n                      epochs=30)\n\n# MixUp augmentation\ndef mixup_data(x, y, alpha=0.4):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n        \n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(device)\n    \n    mixed_x = lam * x + (1 - lam) * x[index]\n    y_a, y_b = y, y[index]\n    return mixed_x, y_a, y_b, lam\n\n# Training setup\nearly_stopping_patience = 20\nbest_val_f1 = 0\nepochs_no_improve = 0\ntrain_losses = []\nval_losses = []\nval_accuracies = []\n\ndef cutmix_data(x, y, alpha=1.0):\n    lam = np.random.beta(alpha, alpha)\n    rand_index = torch.randperm(x.size()[0]).to(device)\n    y_a, y_b = y, y[rand_index]\n\n    bbx1, bby1, bbx2, bby2 = rand_bbox(x.size(), lam)\n    x[:, :, bbx1:bbx2, bby1:bby2] = x[rand_index, :, bbx1:bbx2, bby1:bby2]\n\n    lam = 1 - ((bbx2 - bbx1) * (bby2 - bby1) / (x.size()[-1] * x.size()[-2]))\n    return x, y_a, y_b, lam\n\ndef rand_bbox(size, lam):\n    W = size[2]\n    H = size[3]\n    cut_rat = np.sqrt(1. - lam)\n    cut_w = int(W * cut_rat)\n    cut_h = int(H * cut_rat)\n\n    cx = np.random.randint(W)\n    cy = np.random.randint(H)\n\n    bbx1 = np.clip(cx - cut_w // 2, 0, W)\n    bby1 = np.clip(cy - cut_h // 2, 0, H)\n    bbx2 = np.clip(cx + cut_w // 2, 0, W)\n    bby2 = np.clip(cy + cut_h // 2, 0, H)\n\n    return bbx1, bby1, bbx2, bby2\n\n\ndef predict_tta(model, images, n=5):\n    model.eval()\n    logits = []\n    \n    with torch.no_grad():\n        # Original\n        logits.append(model(images))\n        \n        # Augmented versions\n        for _ in range(n-1):\n            aug_img = torch.stack([\n                train_transform(image=img.cpu().numpy().transpose(1,2,0))['image'] \n                for img in images\n            ]).to(device)\n            logits.append(model(aug_img))\n    \n    return torch.mean(torch.stack(logits), dim=0)\n\n# 4. Modified Training Loop (Add CutMix)\ndef cutmix_data(x, y, alpha=1.0):\n    lam = np.random.beta(alpha, alpha)\n    bx1, bx2, by1, by2 = rand_bbox(x.size(), lam)\n    x[:, :, bx1:bx2, by1:by2] = x.flip(0)[:, :, bx1:bx2, by1:by2]\n    y_a, y_b = y, y.flip(0)\n    return x, y_a, y_b, lam\n\n\nfor epoch in range(200):\n    model.train()\n    running_loss = 0.0\n    \n    for images, labels in tqdm(train_loader, desc=f\"Epoch {epoch+1}\"):\n        images, labels = images.to(device), labels.to(device)\n\n        # MixUp or CutMix\n        if random.random() < 0.5:\n            images, targets_a, targets_b, lam = mixup_data(images, labels)\n        else:\n            images, targets_a, targets_b, lam = cutmix_data(images, labels)\n\n        optimizer.zero_grad()\n\n        with autocast():\n            outputs = model(images)\n            loss = lam * criterion(outputs, targets_a) + (1 - lam) * criterion(outputs, targets_b)\n\n        scaler.scale(loss).backward()\n        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n        scaler.step(optimizer)\n        scaler.update()\n        scheduler.step()\n        ema.update()\n\n        running_loss += loss.item() * images.size(0)\n\n    # Validation\n    model.eval()\n    val_loss = 0.0\n    all_preds, all_labels = [], []\n    \n    with torch.no_grad():\n        with ema.average_parameters():  # use EMA weights\n            for images, labels in valid_loader:\n                images, labels = images.to(device), labels.to(device)\n                outputs = 0\n                for _ in range(3):\n                    if _ > 0:\n                        aug_images = torch.stack([\n                            train_transform(image=img.cpu().numpy().transpose(1,2,0))['image'] \n                            for img in images\n                        ]).to(device)\n                    else:\n                        aug_images = images\n\n                    with autocast():\n                        outputs += model(aug_images)\n\n                outputs /= 3\n                loss = criterion(outputs, labels)\n\n                val_loss += loss.item() * images.size(0)\n                _, preds = torch.max(outputs, 1)\n                all_preds.extend(preds.cpu().numpy())\n                all_labels.extend(labels.cpu().numpy())\n\n    # Metrics\n    train_loss = running_loss / len(train_loader.sampler)\n    val_loss = val_loss / len(valid_loader.dataset)\n    val_acc = accuracy_score(all_labels, all_preds)\n    val_f1 = f1_score(all_labels, all_preds, average='macro')\n\n    # Logging\n    train_losses.append(train_loss)\n    val_losses.append(val_loss)\n    val_accuracies.append(val_acc)\n\n    print(f\"\\nEpoch {epoch+1}:\")\n    print(f\"Train Loss: {train_loss:.4f} | Val Loss: {val_loss:.4f}\")\n    print(f\"Val Accuracy: {val_acc:.4f} | Val F1: {val_f1:.4f}\")\n\n    if val_f1 > best_val_f1:\n        print(f\"Validation F1 Improved ({best_val_f1:.4f} → {val_f1:.4f})\")\n        best_val_f1 = val_f1\n        torch.save({\n            'epoch': epoch,\n            'model_state_dict': model.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'label2id': label2id,\n            'id2label': id2label,\n            'val_f1': val_f1,\n        }, \"best_model.pth\")\n        epochs_no_improve = 0\n    else:\n        epochs_no_improve += 1\n        if epochs_no_improve == early_stopping_patience:\n            print(\"Early stopping triggered!\")\n            break\n\n\nprint(\"Training complete! Best model saved as 'best_model.pth'\")\n\n# Plot training curves\nplt.figure(figsize=(12, 4))\nplt.subplot(1, 2, 1)\nplt.plot(train_losses, label='Train Loss')\nplt.plot(val_losses, label='Val Loss')\nplt.legend()\nplt.subplot(1, 2, 2)\nplt.plot(val_accuracies, label='Val Accuracy')\nplt.legend()\nplt.savefig('training_curves.png')\nplt.close()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport timm\nimport pandas as pd\nfrom PIL import Image\nimport os\nimport numpy as np\nfrom tqdm import tqdm\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\n# Config\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ntest_folder = \"/kaggle/input/2025-bamboo-summer-competiton-dl-pr/test\"\nmodel_path = \"best_model.pth\"\noutput_csv = \"submissions.csv\"\n\n# Load the saved checkpoint\ncheckpoint = torch.load(model_path, map_location=device,weights_only=False)\nlabel2id = checkpoint['label2id']\nid2label = checkpoint['id2label']\nnum_classes = len(label2id)\n\n# Test transforms (same as valid)\ntest_transform = A.Compose([\n    A.Resize(height=256, width=256),\n    A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n    ToTensorV2()\n])\n\n# Recreate model and load weights\nmodel = timm.create_model('convnext_base', pretrained=False)\nmodel.reset_classifier(num_classes)\nmodel.load_state_dict(checkpoint['model_state_dict'], strict=True)\nmodel = model.to(device)\nmodel.eval()\n\n# Prepare image list\nimage_files = sorted(os.listdir(test_folder))\nresults = []\n\n# Inference loop\nfor img_file in tqdm(image_files, desc=\"Predicting\"):\n    img_path = os.path.join(test_folder, img_file)\n    image = Image.open(img_path).convert(\"RGB\")\n    image = test_transform(image=np.array(image))['image'].unsqueeze(0).to(device)\n\n    with torch.no_grad():\n        output = model(image)\n        pred = torch.argmax(output, dim=1).item()\n        label = id2label[pred]\n        results.append({\"filename\": img_file, \"label\": label})\n\n# Save submission\ndf = pd.DataFrame(results)\ndf.to_csv(output_csv, index=False)\nprint(f\"✅ Saved predictions to {output_csv}\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}