{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":667918,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":505654,"modelId":520479}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nfrom torch.utils.data import Dataset, DataLoader\nfrom PIL import Image\nfrom torchvision import transforms\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport numpy as np\nfrom sklearn.preprocessing import StandardScaler\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nfrom torchvision.models import resnet18\nfrom sklearn.metrics import r2_score\nfrom torch.utils.data import random_split\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport copy\n\nBATCH_SIZE_REG = 32\nEPOCHS_REG_HEAD = 100\nEPOCHS_FINETUNE = 100\nPATIENCE = 12\nIMG_DIR = '/kaggle/input/csiro-biomass/'\nDEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\nTARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g']\nTARGETS_CALC=['GDM_g', 'Dry_Total_g']\nLOCAL_RESNET=\"/kaggle/input/resnet18/pytorch/default/1/resnet18-5c106cde.pth\"\n\n# Load your training data\ndf = pd.read_csv(\"/kaggle/input/csiro-biomass/train.csv\")\n\n# Convert long format to wide format\nwide_df = df.pivot_table(\n    index=[\"image_path\", \"Pre_GSHH_NDVI\", \"Height_Ave_cm\", \"Sampling_Date\"],\n    columns=\"target_name\",\n    values=\"target\"\n).reset_index()\nwide_df.columns.name = None\n\n# Drop unnecessary columns\nwide_df = wide_df.drop(columns=['Sampling_Date'])  # omit date\ntab_features = ['Pre_GSHH_NDVI', 'Height_Ave_cm']\n\n# Add Month column from Sampling_Date before dropping\ndf['Sampling_Date'] = pd.to_datetime(df['Sampling_Date'], errors='coerce')\nwide_df['Month'] = df['Sampling_Date'].dt.month\nwide_df['Month'] = wide_df['Month'] / 12.0 \n\n# Normalize tabular features (0-1)\nfor col in tab_features:\n    wide_df[col] = (wide_df[col] - wide_df[col].min()) / (wide_df[col].max() - wide_df[col].min())\n\nwide_df[TARGETS] = wide_df[TARGETS] / 100\n\nprint(\"Tabular features:\", tab_features)\nprint(\"Targets (first 5 rows):\\n\", wide_df[TARGETS].head())\nprint(f\"Total training samples: {len(wide_df)}\")\n\n# regression dataset (images + targets)\nclass BiomassRegressionDataset(Dataset):\n    def __init__(self, df, img_dir, targets, tab_cols, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.targets = torch.tensor(df[targets].values, dtype=torch.float32)\n        self.tabular = torch.tensor(df[tab_cols].values, dtype=torch.float32)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.img_dir, row['image_path'])\n        img = Image.open(img_path).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n        return img, self.tabular[idx], self.targets[idx]\n\n# Test Time Augmentation transforms (tensor-based for efficiency)\ndef create_tta_functions():\n    return [\n        lambda x: x,  # Identity (no transform)\n        lambda x: torch.flip(x, [-1]),  # Horizontal flip\n        lambda x: transforms.functional.adjust_brightness(x, 1.1),  # Brightness up\n        lambda x: transforms.functional.adjust_brightness(x, 0.9),  # Brightness down\n    ]\n\n# Simple regression model with ResNet18 using local weights\nclass SimpleRegressor(nn.Module):\n    def __init__(self, output_dim=3):\n        super().__init__()\n        # Load ResNet18 with local weights\n        self.backbone = resnet18()\n        # Load state_dict from local file\n        if LOCAL_RESNET and os.path.exists(LOCAL_RESNET):\n            sd = torch.load(LOCAL_RESNET, map_location='cpu', weights_only=False)\n            # handle possible \"state_dict\" key or raw dict\n            if 'state_dict' in sd:\n                sd = sd['state_dict']\n            # if keys have \"module.\" prefix, strip them\n            new_sd = {}\n            for k, v in sd.items():\n                kk = k\n                if k.startswith('module.'):\n                    kk = k[len('module.'):]\n                new_sd[kk] = v\n            self.backbone.load_state_dict(new_sd, strict=False)\n            print(\"Loaded local ResNet weights from\", LOCAL_RESNET)\n        else:\n            print(\"Local resnet weights not found; using random init (offline mode).\")\n        \n        self.backbone.fc = nn.Identity()  # Remove classifier\n        # Simple linear head\n        self.head = nn.Linear(512, output_dim)\n    \n    def forward(self, x):\n        features = self.backbone(x)  # (B, 512)\n        return self.head(features)   # (B, 3)\n\n# Create multiple models for ensemble - ALL USE SAME DATA, DIFFERENT SEEDS\nmodels = []\noptimizers = []\n\nreg_transform = transforms.Compose([\n    transforms.Resize((224,224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225])\n])\n\n# Use ALL data for all models (best for 300 images)\ntrain_dataset = BiomassRegressionDataset(\n    wide_df, IMG_DIR, TARGETS, tab_features, transform=reg_transform\n)\ntrain_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE_REG, shuffle=True, num_workers=4, pin_memory=True)\n\nfor i in range(5):\n    print(f\"\\nTraining Model {i + 1}/5 with different random seed\")\n    \n    # Set different random seed for each model to get different initial weights\n    torch.manual_seed(42 + i)  # Different seed for each model\n    np.random.seed(42 + i)\n    \n    # Initialize model with different random initialization\n    model = SimpleRegressor(output_dim=3).to(DEVICE)\n    \n    # Different random initialization for the head (this makes models different)\n    torch.nn.init.xavier_uniform_(model.head.weight)\n    torch.nn.init.zeros_(model.head.bias)\n    \n    models.append(model)\n    \n    # Different learning rates for different parts\n    backbone_params = [p for n,p in model.backbone.named_parameters() if p.requires_grad]\n    head_params = [p for p in model.head.parameters()]\n    \n    optimizer = torch.optim.AdamW([\n        {'params': head_params, 'lr': 1e-3},\n        {'params': backbone_params, 'lr': 1e-5}\n    ], weight_decay=1e-4)\n    optimizers.append(optimizer)\n    \n    # Phase 1: Train only the head\n    print(\"  Phase 1: Training head only\")\n    for param in model.backbone.parameters():\n        param.requires_grad = False\n    for param in model.head.parameters():\n        param.requires_grad = True\n    \n    best_reg_loss = float('inf')\n    counter = 0\n    for epoch in range(EPOCHS_REG_HEAD // 2):  # Fewer epochs for head\n        model.train()\n        tot = 0.0\n        for imgs, tab, targets in train_loader:\n            imgs, tab, targets = imgs.to(DEVICE), tab.to(DEVICE), targets.to(DEVICE)\n            preds = model(imgs)\n            loss = F.mse_loss(preds, targets)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            tot += loss.item()\n        avg = tot / len(train_loader)\n        print(f\"    Head epoch {epoch+1}/{EPOCHS_REG_HEAD//2}  loss={avg:.4f}\")\n        if avg < best_reg_loss:\n            best_reg_loss = avg\n            counter = 0\n        else:\n            counter += 1\n            if counter >= PATIENCE // 2:\n                print(\"    Early stopping head training.\")\n                break\n    \n    # Phase 2: Fine-tune backbone with frozen head\n    print(\"  Phase 2: Fine-tuning backbone\")\n    for param in model.backbone.layer4.parameters():\n        param.requires_grad = True\n    for param in model.backbone.layer3.parameters():\n        param.requires_grad = True\n    for param in model.head.parameters():\n        param.requires_grad = True  # Keep head trainable\n    \n    # Adjust optimizer for fine-tuning\n    backbone_params = [p for n,p in model.backbone.named_parameters() if p.requires_grad]\n    head_params = [p for p in model.head.parameters()]\n    \n    optimizer = torch.optim.AdamW([\n        {'params': head_params, 'lr': 1e-4},\n        {'params': backbone_params, 'lr': 1e-6}  # Much smaller LR for backbone\n    ], weight_decay=1e-4)\n    \n    best_ft = float('inf')\n    counter = 0\n    for epoch in range(EPOCHS_FINETUNE):\n        model.train()\n        tot = 0.0\n        for imgs, tab, targets in train_loader:\n            imgs, tab, targets = imgs.to(DEVICE), tab.to(DEVICE), targets.to(DEVICE)\n            preds = model(imgs)\n            loss = F.mse_loss(preds, targets)\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n            tot += loss.item()\n        avg = tot / len(train_loader)\n        print(f\"    Finetune epoch {epoch+1}/{EPOCHS_FINETUNE}  loss={avg:.4f}\")\n        if avg < best_ft:\n            best_ft = avg\n            counter = 0\n            # Save this specific model\n            torch.save(model.state_dict(), f\"reg_model_{i}_best.pth\")\n        else:\n            counter += 1\n            if counter >= PATIENCE:\n                print(\"    Early stopping finetune.\")\n                break\n\nprint(\"\\nTraining completed. All models saved.\")\n\n# Load all trained models for inference\ninference_models = []\nfor i in range(5):\n    model = SimpleRegressor(output_dim=3).to(DEVICE)\n    model.load_state_dict(torch.load(f\"reg_model_{i}_best.pth\"))\n    model.eval()\n    inference_models.append(model)\n\nprint(f\"\\nLoaded {len(inference_models)} models for ensemble prediction\")\n\n# Efficient TTA + Ensemble function\n@torch.no_grad()\ndef predict_with_tta_ensemble_batch(images, models, tta_funcs):\n    \"\"\"\n    Tensor-based TTA for efficiency\n    \"\"\"\n    batch_size = images.size(0)\n    all_predictions = []\n    \n    for tta_func in tta_funcs:\n        # Apply transform to entire batch at once\n        tta_batch = tta_func(images)\n        \n        # Get predictions from all models\n        model_predictions = []\n        for model in models:\n            model.eval()\n            pred = model(tta_batch)\n            model_predictions.append(pred)\n        \n        # Average across models\n        avg_pred = torch.stack(model_predictions).mean(dim=0)\n        all_predictions.append(avg_pred)\n    \n    # Average across TTA transforms\n    final_pred = torch.stack(all_predictions).mean(dim=0)\n    return final_pred\n\nclass TestDataset(Dataset):\n    def __init__(self, df, img_dir, transform=None):\n        self.df = df.reset_index(drop=True)\n        self.img_dir = img_dir\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.img_dir, row['image_path'])\n        img = Image.open(img_path).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n        return img, row['image_path']\n\n# Load test CSV\ntest_df = pd.read_csv(\"/kaggle/input/csiro-biomass/test.csv\")\ntest_df = test_df.drop_duplicates(subset=[\"image_path\"]).reset_index(drop=True)\n\ntest_transform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225]),\n])\n\n# Use smaller batch size for inference to avoid memory issues\nBATCH_SIZE_INFERENCE = 32  # Reduced from 128\ntest_dataset = TestDataset(test_df, IMG_DIR, transform=test_transform)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE_INFERENCE, shuffle=False)\n\n# Use tensor-based TTA functions\ntta_functions = create_tta_functions()\n\n# Inference with ensemble and TTA\nresults = []\n\nwith torch.no_grad():\n    for images, paths in test_loader:\n        images = images.to(DEVICE)\n        \n        # Use efficient TTA + Ensemble\n        preds = predict_with_tta_ensemble_batch(images, inference_models, tta_functions)\n        preds = preds.cpu().numpy()\n\n        # invert normalization to real biomass units\n        pred_real = preds * 100\n        pred_real = np.maximum(0, pred_real)\n        dry_green  = pred_real[:, 0]\n        dry_dead   = pred_real[:, 1]\n        dry_clover = pred_real[:, 2]\n\n        gdm       = dry_clover + dry_green\n        dry_total = dry_clover + dry_dead + dry_green\n\n        # build submission rows\n        for i, path in enumerate(paths):\n            base_id = os.path.splitext(os.path.basename(path))[0]\n\n            results.extend([\n                {'sample_id': f\"{base_id}__Dry_Clover_g\", 'target': float(dry_clover[i])},\n                {'sample_id': f\"{base_id}__Dry_Dead_g\",   'target': float(dry_dead[i])},\n                {'sample_id': f\"{base_id}__Dry_Green_g\",  'target': float(dry_green[i])},\n                {'sample_id': f\"{base_id}__Dry_Total_g\",  'target': float(dry_total[i])},\n                {'sample_id': f\"{base_id}__GDM_g\",        'target': float(gdm[i])},\n            ])\n\n# Save CSV for Kaggle submission\nsubmission_df = pd.DataFrame(results)\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\n\nprint(\"Submission saved to /kaggle/working/submission.csv\")\nprint(submission_df.head())\n\n# Save all models\nfor i, model in enumerate(models):\n    torch.save(model.state_dict(), f\"simple_reg_model_{i}.pth\")\n\nprint(\"\\nAll models saved successfully!\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-17T23:31:21.432862Z","iopub.execute_input":"2025-12-17T23:31:21.433070Z","iopub.status.idle":"2025-12-17T23:34:56.444740Z","shell.execute_reply.started":"2025-12-17T23:31:21.433053Z","shell.execute_reply":"2025-12-17T23:34:56.443526Z"}},"outputs":[],"execution_count":null}]}