{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":274726649,"sourceType":"kernelVersion"}],"dockerImageVersionId":31154,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nimport timm # We need timm for efficientnet\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.model_selection import KFold\nfrom sklearn.decomposition import PCA\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n# --- Configuration ---\nTRAIN_CSV_PATH = \"/kaggle/input/csiro-biomass/train.csv\"\nBASE_DATA_DIR = \"/kaggle/input/csiro-biomass/\"\n\nclass CFG:\n    IMAGE_SIZE = 384 # CNNs like slightly larger inputs for texture\n    # SWITCHING MODEL: EfficientNet is king of small data\n    MODEL_NAME = \"tf_efficientnet_b0_ns\" \n    BATCH_SIZE = 16 # Smaller batch for better generalization on small data\n    LEARNING_RATE = 1e-3 # Higher initial LR because we are only training the head\n    WEIGHT_DECAY = 1e-4\n    EPOCHS = 20\n    TARGET_COLS = [\"Dry_Total_g\", \"Dry_Green_g\", \"Dry_Dead_g\", \"Dry_Clover_g\", \"GDM_g\"]\n    NUM_TARGETS = len(TARGET_COLS)\n    DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n    SEED = 42\n    N_FOLDS = 5\n\n# --- Seed ---\ndef set_seed(seed=CFG.SEED):\n    torch.manual_seed(seed)\n    np.random.seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seed()\n\n# --- Dataset ---\ntrain_transforms = T.Compose([\n    T.Resize((CFG.IMAGE_SIZE, CFG.IMAGE_SIZE)),\n    T.RandomHorizontalFlip(),\n    T.RandomVerticalFlip(),\n    # Reduced ColorJitter: Small datasets get confused by strong jitter\n    T.ColorJitter(brightness=0.1, contrast=0.1), \n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nvalid_transforms = T.Compose([\n    T.Resize((CFG.IMAGE_SIZE, CFG.IMAGE_SIZE)),\n    T.ToTensor(),\n    T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\nclass GrassDataset(Dataset):\n    def __init__(self, df, target_cols, transforms=None):\n        self.df = df\n        self.transforms = transforms\n        self.target_cols = target_cols\n        \n    def __len__(self):\n        return len(self.df)\n        \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        image_path = os.path.join(BASE_DATA_DIR, row[\"image_path\"])\n        image = Image.open(image_path).convert(\"RGB\")\n        if self.transforms:\n            image = self.transforms(image)\n        targets = torch.tensor(row[self.target_cols].values.astype(np.float32))\n        return image, targets\n\n# --- The \"Small Data\" Model ---\nclass EfficientRegressor(nn.Module):\n    def __init__(self, model_name=CFG.MODEL_NAME, num_targets=CFG.NUM_TARGETS):\n        super().__init__()\n        print(f\"Loading {model_name} (Pretrained)...\")\n        # Load backbone, remove original head (num_classes=0)\n        self.backbone = timm.create_model(model_name, pretrained=True, num_classes=0)\n        \n        # FREEZE THE BACKBONE: This is crucial for 357 images\n        for param in self.backbone.parameters():\n            param.requires_grad = False\n            \n        in_features = self.backbone.num_features\n        \n        # A simple regressor head\n        self.head = nn.Sequential(\n            nn.Dropout(0.5), # High dropout for regularization\n            nn.Linear(in_features, 256),\n            nn.ReLU(),\n            nn.Dropout(0.2),\n            nn.Linear(256, num_targets)\n        )\n        \n    def forward(self, x):\n        features = self.backbone(x)\n        return self.head(features)\n    \n    def unfreeze_backbone(self):\n        print(\"🔓 Unfreezing Backbone layers for fine-tuning...\")\n        for param in self.backbone.parameters():\n            param.requires_grad = True\n\n# --- Diagnostics (PCA) ---\ndef check_manifold(model, loader, device):\n    print(\"\\n[DIAGNOSTIC] Checking Feature Manifold (PCA)...\")\n    model.eval()\n    features = []\n    targets_sum = []\n    \n    with torch.no_grad():\n        for images, targets in loader:\n            images = images.to(device)\n            # Get features from backbone directly\n            feats = model.backbone(images)\n            features.append(feats.cpu().numpy())\n            targets_sum.append(targets.sum(dim=1).numpy())\n            \n    features = np.vstack(features)\n    targets_sum = np.concatenate(targets_sum)\n    \n    pca = PCA(n_components=2)\n    embedding = pca.fit_transform(features)\n    \n    plt.figure(figsize=(8, 6))\n    sc = plt.scatter(embedding[:, 0], embedding[:, 1], c=targets_sum, cmap='viridis', s=20, alpha=0.8)\n    plt.colorbar(sc, label='Biomass')\n    plt.title(f\"Feature Space (Frozen Backbone) - PCA Var: {np.sum(pca.explained_variance_ratio_):.2f}\")\n    plt.xlabel(\"PC1\")\n    plt.ylabel(\"PC2\")\n    plt.show()\n    \n    # Auto-inference\n    var = np.sum(pca.explained_variance_ratio_)\n    if var > 0.95:\n        print(\"✅ Good! The pre-trained weights see clear differences in the images.\")\n    else:\n        print(\"⚠️ Caution: The pre-trained weights don't naturally separate these images well.\")\n\n# --- Training Loop ---\ndef train_one_epoch(loader, model, optimizer, criterion):\n    model.train()\n    total_loss = 0.0\n    for images, targets in loader:\n        images, targets = images.to(CFG.DEVICE), targets.to(CFG.DEVICE)\n        preds = model(images)\n        loss = criterion(preds, targets)\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n        total_loss += loss.item()\n    return total_loss / len(loader)\n\ndef validate(loader, model):\n    model.eval()\n    preds_all, targets_all = [], []\n    with torch.no_grad():\n        for images, targets in loader:\n            images, targets = images.to(CFG.DEVICE), targets.to(CFG.DEVICE)\n            preds = model(images)\n            preds_all.append(preds.cpu().numpy())\n            targets_all.append(targets.cpu().numpy())\n    \n    preds_all = np.vstack(preds_all)\n    targets_all = np.vstack(targets_all)\n    \n    # R2 Score\n    ss_res = np.sum((targets_all - preds_all) ** 2)\n    ss_tot = np.sum((targets_all - np.mean(targets_all, axis=0)) ** 2)\n    r2 = 1 - ss_res / (ss_tot + 1e-6)\n    return r2, preds_all, targets_all\n\ndef run_training():\n    print(\"--- CONFIGURATION: Log-Transformed Targets + TTA ---\")\n    df_long = pd.read_csv(TRAIN_CSV_PATH)\n    df_wide = df_long.pivot(index=\"image_path\", columns=\"target_name\", values=\"target\").reset_index()\n    \n    kf = KFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n    \n    for fold, (train_idx, val_idx) in enumerate(kf.split(df_wide)):\n        print(f\"\\n=== FOLD {fold+1} ===\")\n        \n        train_df = df_wide.iloc[train_idx].copy().reset_index(drop=True)\n        val_df = df_wide.iloc[val_idx].copy().reset_index(drop=True)\n        \n        # --- CRITICAL CHANGE: LOG TRANSFORM TARGETS ---\n        # We train on Log, but we calculate R2 on original scale\n        print(\"Applying Log1p to Training Targets...\")\n        train_targets_orig = train_df[CFG.TARGET_COLS].values.astype(np.float32)\n        val_targets_orig = val_df[CFG.TARGET_COLS].values.astype(np.float32)\n        \n        # Fit scaler on LOG values\n        scaler = StandardScaler()\n        train_targets_log = np.log1p(train_targets_orig)\n        val_targets_log = np.log1p(val_targets_orig)\n        \n        train_df[CFG.TARGET_COLS] = scaler.fit_transform(train_targets_log)\n        val_df[CFG.TARGET_COLS] = scaler.transform(val_targets_log)\n        \n        # Loaders\n        train_loader = DataLoader(GrassDataset(train_df, CFG.TARGET_COLS, train_transforms), \n                                  batch_size=CFG.BATCH_SIZE, shuffle=True, num_workers=2)\n        val_loader = DataLoader(GrassDataset(val_df, CFG.TARGET_COLS, valid_transforms), \n                                batch_size=CFG.BATCH_SIZE, shuffle=False, num_workers=2)\n        \n        model = EfficientRegressor().to(CFG.DEVICE)\n        criterion = nn.MSELoss()\n        \n        # --- Phase 1: Head Only ---\n        print(\"Phase 1: Linear Probing (Log Space)...\")\n        optimizer = optim.Adam(model.head.parameters(), lr=1e-3)\n        for epoch in range(10):\n            loss = train_one_epoch(train_loader, model, optimizer, criterion)\n            # Monitor Loss only (R2 requires inverse transform, calculated below)\n            if (epoch+1) % 5 == 0:\n                print(f\"  Ep {epoch+1} | Log-Space Loss: {loss:.4f}\")\n\n        # --- Phase 2: Fine Tuning ---\n        print(\"Phase 2: Fine Tuning...\")\n        model.unfreeze_backbone()\n        optimizer = optim.AdamW([\n            {'params': model.backbone.parameters(), 'lr': 5e-5},\n            {'params': model.head.parameters(), 'lr': 5e-4}\n        ], weight_decay=1e-3)\n        \n        scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=3)\n        \n        best_r2 = -np.inf\n        \n        for epoch in range(20):\n            train_loss = train_one_epoch(train_loader, model, optimizer, criterion)\n            \n            # --- VALIDATION with TTA and INVERSE TRANSFORM ---\n            model.eval()\n            preds_orig_scale = []\n            targets_orig_scale = []\n            \n            with torch.no_grad():\n                for images, targets in val_loader:\n                    images = images.to(CFG.DEVICE)\n                    \n                    # TTA: Average (Image) + (Flipped Image)\n                    out1 = model(images)\n                    out2 = model(torch.flip(images, dims=[3])) # Horizontal Flip\n                    preds_log_scaled = (out1 + out2) / 2.0\n                    \n                    # 1. Inverse Scale\n                    preds_log = scaler.inverse_transform(preds_log_scaled.cpu().numpy())\n                    \n                    # 2. Inverse Log (Expm1)\n                    preds_orig = np.expm1(preds_log)\n                    \n                    # 3. Clip negative predictions (Biomass can't be negative)\n                    preds_orig = np.maximum(0, preds_orig)\n                    \n                    preds_orig_scale.append(preds_orig)\n                    # Retrieve original un-transformed targets for fair comparison\n                    # We can't use 'targets' from loader because those are scaled logs\n            \n            # Re-align with original dataframe order for scoring\n            preds_final = np.vstack(preds_orig_scale)\n            targets_final = val_targets_orig # Use the saved original values\n            \n            # Calculate Real R2\n            ss_res = np.sum((targets_final - preds_final) ** 2)\n            ss_tot = np.sum((targets_final - np.mean(targets_final, axis=0)) ** 2)\n            val_r2 = 1 - ss_res / (ss_tot + 1e-6)\n            \n            scheduler.step(train_loss)\n            print(f\"  Ep {epoch+11} | Loss: {train_loss:.4f} | Real Scale R2: {val_r2:.4f}\")\n            \n            if val_r2 > best_r2:\n                best_r2 = val_r2\n                print(f\"    --> New Best R2: {best_r2:.4f}\")\n                # Optional: Check Manifold again if score is high\n                if val_r2 > 0.55:\n                     check_manifold(model, val_loader, CFG.DEVICE)\n\n        break # Stop after 1 fold\n\nif __name__ == \"__main__\":\n    run_training()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-21T17:41:25.835435Z","iopub.execute_input":"2025-11-21T17:41:25.835789Z","iopub.status.idle":"2025-11-21T17:47:58.738537Z","shell.execute_reply.started":"2025-11-21T17:41:25.835763Z","shell.execute_reply":"2025-11-21T17:47:58.737598Z"}},"outputs":[],"execution_count":null}]}