{"cells":[{"cell_type":"markdown","metadata":{},"source":"# CSIRO Biomass Prediction - Iteration 4: ResNet50\n\n**Strategy:** Switch to ResNet50 architecture for better training from scratch\n\n**Key Changes:**\n- Architecture: ResNet50 (vs EfficientNet-B1)\n- Image size: 384x384 (vs 320x320)\n- Training: 30 epochs with cosine annealing\n- Goal: Break through performance ceiling with different architecture"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom sklearn.model_selection import train_test_split\nimport warnings\nwarnings.filterwarnings('ignore')\n\nprint(f\"PyTorch version: {torch.__version__}\")\nprint(f\"CUDA available: {torch.cuda.is_available()}\")\nif torch.cuda.is_available():\n    print(f\"CUDA device: {torch.cuda.get_device_name(0)}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Configuration\nIMG_SIZE = 384\nBATCH_SIZE = 10\nEPOCHS = 30\nLEARNING_RATE = 1e-3\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nSEED = 42\n\n# Set seeds\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\n\nprint(f\"Configuration:\")\nprint(f\"- Image size: {IMG_SIZE}x{IMG_SIZE}\")\nprint(f\"- Batch size: {BATCH_SIZE}\")\nprint(f\"- Epochs: {EPOCHS}\")\nprint(f\"- Learning rate: {LEARNING_RATE}\")\nprint(f\"- Device: {DEVICE}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Load data\ntrain_df = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntest_df = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\nsample_submission = pd.read_csv('/kaggle/input/csiro-biomass/sample_submission.csv')\n\nprint(f\"Training samples: {len(train_df)}\")\nprint(f\"Test samples: {len(test_df)}\")\nprint(f\"\\nTrain columns: {train_df.columns.tolist()}\")\nprint(f\"Test columns: {test_df.columns.tolist()}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Dataset class\nclass BiomassDataset(Dataset):\n    def __init__(self, df, base_path, transform=None, is_test=False):\n        self.df = df\n        self.base_path = base_path\n        self.transform = transform\n        self.is_test = is_test\n        \n        if not is_test:\n            self.targets = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.base_path, row['image_path'])\n        \n        # Load image\n        image = Image.open(img_path).convert('RGB')\n        \n        if self.transform:\n            image = self.transform(image)\n        \n        if self.is_test:\n            return image, row['sample_id'], row['target_name']\n        else:\n            # Get target value\n            target_name = row['target_name']\n            target_value = row[target_name]\n            return image, torch.tensor([target_value], dtype=torch.float32)\n\nprint(\"Dataset class defined\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Data transforms - moderate augmentation\ntrain_transform = transforms.Compose([\n    transforms.RandomResizedCrop(IMG_SIZE, scale=(0.8, 1.0)),\n    transforms.RandomHorizontalFlip(p=0.5),\n    transforms.RandomRotation(15),\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nval_transform = transforms.Compose([\n    transforms.Resize((IMG_SIZE, IMG_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n])\n\nprint(\"Transforms defined\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Split data\ntrain_data, val_data = train_test_split(train_df, test_size=0.15, random_state=SEED)\n\ntrain_dataset = BiomassDataset(train_data, '/kaggle/input/csiro-biomass', transform=train_transform)\nval_dataset = BiomassDataset(val_data, '/kaggle/input/csiro-biomass', transform=val_transform)\ntest_dataset = BiomassDataset(test_df, '/kaggle/input/csiro-biomass', transform=val_transform, is_test=True)\n\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\nval_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\ntest_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=2)\n\nprint(f\"Training batches: {len(train_loader)}\")\nprint(f\"Validation batches: {len(val_loader)}\")\nprint(f\"Test samples: {len(test_dataset)}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# ResNet50 Model\nclass ResNet50Regressor(nn.Module):\n    def __init__(self, num_outputs=1):\n        super(ResNet50Regressor, self).__init__()\n        # Load ResNet50 without pretrained weights (internet disabled)\n        self.resnet = models.resnet50(weights=None)\n        \n        # Remove the final FC layer\n        num_features = self.resnet.fc.in_features\n        self.resnet.fc = nn.Identity()\n        \n        # Custom regression head\n        self.regressor = nn.Sequential(\n            nn.Linear(num_features, 1024),\n            nn.BatchNorm1d(1024),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(1024, 512),\n            nn.BatchNorm1d(512),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(512, num_outputs)\n        )\n    \n    def forward(self, x):\n        features = self.resnet(x)\n        output = self.regressor(features)\n        return output\n\nmodel = ResNet50Regressor(num_outputs=1).to(DEVICE)\nprint(f\"Model created with {sum(p.numel() for p in model.parameters()):,} parameters\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Loss, optimizer, scheduler\ncriterion = nn.MSELoss()\noptimizer = optim.AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=1e-4)\nscheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=EPOCHS)\n\nprint(\"Training setup complete\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Training function\ndef train_epoch(model, loader, criterion, optimizer, device):\n    model.train()\n    total_loss = 0\n    \n    for images, targets in loader:\n        images = images.to(device)\n        targets = targets.to(device)\n        \n        optimizer.zero_grad()\n        outputs = model(images)\n        loss = criterion(outputs, targets)\n        loss.backward()\n        \n        # Gradient clipping\n        torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n        \n        optimizer.step()\n        total_loss += loss.item()\n    \n    return total_loss / len(loader)\n\ndef validate(model, loader, criterion, device):\n    model.eval()\n    total_loss = 0\n    \n    with torch.no_grad():\n        for images, targets in loader:\n            images = images.to(device)\n            targets = targets.to(device)\n            \n            outputs = model(images)\n            loss = criterion(outputs, targets)\n            total_loss += loss.item()\n    \n    return total_loss / len(loader)\n\nprint(\"Training functions defined\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Training loop\nprint(\"\\nStarting training...\")\nstart_time = time.time()\n\nbest_val_loss = float('inf')\npatience_counter = 0\npatience = 10\n\nfor epoch in range(EPOCHS):\n    epoch_start = time.time()\n    \n    train_loss = train_epoch(model, train_loader, criterion, optimizer, DEVICE)\n    val_loss = validate(model, val_loader, criterion, DEVICE)\n    \n    scheduler.step()\n    \n    epoch_time = time.time() - epoch_start\n    \n    print(f\"Epoch {epoch+1}/{EPOCHS} - Train Loss: {train_loss:.4f} - Val Loss: {val_loss:.4f} - Time: {epoch_time:.1f}s - LR: {scheduler.get_last_lr()[0]:.6f}\")\n    \n    # Early stopping\n    if val_loss < best_val_loss:\n        best_val_loss = val_loss\n        patience_counter = 0\n        # Save best model\n        torch.save(model.state_dict(), 'best_model.pth')\n    else:\n        patience_counter += 1\n        if patience_counter >= patience:\n            print(f\"\\nEarly stopping at epoch {epoch+1}\")\n            break\n\ntotal_time = time.time() - start_time\nprint(f\"\\nTraining completed in {total_time/60:.2f} minutes\")\nprint(f\"Best validation loss: {best_val_loss:.4f}\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Load best model\nmodel.load_state_dict(torch.load('best_model.pth'))\nmodel.eval()\nprint(\"Best model loaded for inference\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Generate predictions\nprint(\"\\nGenerating predictions...\")\npredictions = []\n\nwith torch.no_grad():\n    for images, sample_ids, target_names in test_loader:\n        images = images.to(DEVICE)\n        outputs = model(images)\n        \n        for i in range(len(sample_ids)):\n            predictions.append({\n                'sample_id': sample_ids[i],\n                'target_name': target_names[i],\n                'prediction': outputs[i].cpu().item()\n            })\n\nprint(f\"Generated {len(predictions)} predictions\")"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Create submission\nsubmission_df = pd.DataFrame(predictions)\nsubmission_df['id'] = submission_df['sample_id'] + '_' + submission_df['target_name']\nsubmission_df = submission_df[['id', 'prediction']]\n\n# Ensure non-negative predictions\nsubmission_df['prediction'] = submission_df['prediction'].clip(lower=0)\n\nprint(\"\\nSubmission preview:\")\nprint(submission_df.head(10))\nprint(f\"\\nPrediction statistics:\")\nprint(submission_df['prediction'].describe())"},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":"# Save submission\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)\nprint(\"\\n✅ Submission saved to /kaggle/working/submission.csv\")\nprint(f\"Total training time: {total_time/60:.2f} minutes\")\nprint(f\"Best validation loss: {best_val_loss:.4f}\")"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"}},"nbformat":4,"nbformat_minor":4}