{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'\n\nimport gc\nimport pandas as pd\nimport numpy as np\nimport cv2\nfrom pathlib import Path\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.metrics import r2_score\nimport albumentations as A\n\n# Clear GPU\ntorch.cuda.empty_cache()\ngc.collect()\n\n# Set seed\nSEED = 42\ntorch.manual_seed(SEED)\nnp.random.seed(SEED)\nos.environ['PYTHONHASHSEED'] = str(SEED)\nif torch.cuda.is_available():\n    torch.cuda.manual_seed(SEED)\n    torch.cuda.manual_seed_all(SEED)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\n\n# --- Global Settings ---\nDATA_PATH = Path('/kaggle/input/csiro-biomass')\nIMG_SIZE = 224\nBATCH_SIZE = 16 # Keep small for ResNet memory\nTARGETS = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nTARGET_WEIGHTS = torch.tensor([0.1, 0.1, 0.1, 0.2, 0.5], dtype=torch.float32)\n\n# --- Data Loading ---\ntrain_df = pd.read_csv(DATA_PATH/'train.csv')\ntest_df  = pd.read_csv(DATA_PATH/'test.csv')\n\ntrain_df['image_id'] = train_df['image_path'].map(lambda x: Path(x).stem)\ntest_df['image_id']  = test_df['image_path'].map(lambda x: Path(x).stem)\ntrain_df['target_name'] = train_df['sample_id'].str.split('__').str[-1]\ntest_df['target_name'] = test_df['sample_id'].str.split('__').str[-1]\n\n# Pivot training data\ntrain_pivot = train_df.pivot(index='image_id', columns='target_name', values='target').reset_index()\ntrain_pivot.columns.name = None\ntrain_meta = train_df.drop_duplicates('image_id').set_index('image_id').reset_index()\ntrain_meta = pd.merge(train_meta, train_pivot, on='image_id')\ntest_meta = test_df.drop_duplicates('image_id').set_index('image_id').reset_index()\n\n# Feature Engineering\ndef add_features(df):\n    df = df.copy()\n    # Explicit checks to avoid AttributeError\n    if 'Pre_GSHH_NDVI' in df.columns:\n        df['ndvi'] = pd.to_numeric(df['Pre_GSHH_NDVI'], errors='coerce').fillna(0.0)\n    else:\n        df['ndvi'] = 0.0\n        \n    if 'Height_Ave_cm' in df.columns:\n        df['height'] = pd.to_numeric(df['Height_Ave_cm'], errors='coerce').fillna(30.0)\n    else:\n        df['height'] = 30.0\n    \n    if 'Sampling_Date' in df.columns:\n        dates = pd.to_datetime(df['Sampling_Date'], errors='coerce')\n        df['doy'] = dates.dt.dayofyear.fillna(180)\n        df['month'] = dates.dt.month.fillna(6)\n    else:\n        df['doy'], df['month'] = 180.0, 6.0\n    \n    df['sin_doy'] = np.sin(2 * np.pi * df['doy'] / 365)\n    df['cos_doy'] = np.cos(2 * np.pi * df['doy'] / 365)\n    df['State'] = df['State'].fillna('Unknown') if 'State' in df.columns else 'Unknown'\n    df['Species'] = df['Species'].fillna('Unknown') if 'Species' in df.columns else 'Unknown'\n    df['ndvi_x_height'] = df['ndvi'] * df['height']\n    return df\n\ntrain_meta = add_features(train_meta)\ntest_meta = add_features(test_meta)\n\n# Preprocessing\ntabular_cols = ['ndvi','height','sin_doy','cos_doy','ndvi_x_height','month','State','Species']\nnumeric_cols = [c for c in tabular_cols if c not in ['State','Species']]\ncategorical_cols = ['State','Species']\n\npreprocessor = ColumnTransformer([\n    ('num', StandardScaler(), numeric_cols),\n    ('cat', OneHotEncoder(handle_unknown='ignore', sparse_output=False), categorical_cols)\n])\n\nX_train_tab = preprocessor.fit_transform(train_meta[tabular_cols])\nX_test_tab = preprocessor.transform(test_meta[tabular_cols])\n\n# Targets\nY_train_log = np.log1p(train_meta[TARGETS].values).astype(np.float32)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T10:09:23.672810Z","iopub.execute_input":"2025-11-24T10:09:23.673111Z","iopub.status.idle":"2025-11-24T10:10:01.944881Z","shell.execute_reply.started":"2025-11-24T10:09:23.673092Z","shell.execute_reply":"2025-11-24T10:10:01.944015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_train_transforms(size):\n    return A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.15, rotate_limit=45, p=0.5, \n                           border_mode=cv2.BORDER_REFLECT),\n        A.OneOf([\n            A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2),\n            A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10),\n        ], p=0.7),\n        A.GaussNoise(var_limit=(10.0, 30.0), p=0.2),\n        A.Resize(size, size),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ])\n\ndef get_test_transforms(size):\n    return A.Compose([\n        A.Resize(size, size),\n        A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),\n    ])\n\nclass BiomassDataset(Dataset):\n    def __init__(self, meta_df, tabular_data, is_train=True, targets=None, transforms=None):\n        self.meta_df = meta_df.reset_index(drop=True)\n        self.tabular_data = tabular_data\n        self.is_train = is_train\n        self.targets = targets\n        self.transforms = transforms\n        self.folder = DATA_PATH / ('train' if is_train else 'test')\n    \n    def __len__(self):\n        return len(self.meta_df)\n    \n    def __getitem__(self, idx):\n        img_path = str(self.folder / (self.meta_df.iloc[idx]['image_id'] + '.jpg'))\n        image = cv2.imread(img_path)\n        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n        \n        if self.transforms:\n            image = self.transforms(image=image)['image']\n        image = image.transpose(2, 0, 1).astype(np.float32)\n        tabular = self.tabular_data[idx].astype(np.float32)\n        \n        if self.targets is not None:\n            return torch.from_numpy(image), torch.from_numpy(tabular), torch.from_numpy(self.targets[idx])\n        return torch.from_numpy(image), torch.from_numpy(tabular)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T10:10:01.946146Z","iopub.execute_input":"2025-11-24T10:10:01.946360Z","iopub.status.idle":"2025-11-24T10:10:01.957159Z","shell.execute_reply.started":"2025-11-24T10:10:01.946343Z","shell.execute_reply":"2025-11-24T10:10:01.956394Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- Advanced Architecture Components ---\n\nclass SEBlock(nn.Module):\n    \"\"\"Squeeze-and-Excitation Block for channel attention\"\"\"\n    def __init__(self, in_channels, reduction=16):\n        super().__init__()\n        self.avg_pool = nn.AdaptiveAvgPool2d(1)\n        self.fc = nn.Sequential(\n            nn.Linear(in_channels, in_channels // reduction, bias=False),\n            nn.ReLU(inplace=True),\n            nn.Linear(in_channels // reduction, in_channels, bias=False),\n            nn.Sigmoid()\n        )\n\n    def forward(self, x):\n        b, c, _, _ = x.size()\n        y = self.avg_pool(x).view(b, c)\n        y = self.fc(y).view(b, c, 1, 1)\n        return x * y.expand_as(x)\n\nclass ResBlock(nn.Module):\n    \"\"\"Standard Residual Block with optional downsampling and SE\"\"\"\n    def __init__(self, in_channels, out_channels, stride=1, use_se=True):\n        super().__init__()\n        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n        self.bn1 = nn.BatchNorm2d(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n        self.bn2 = nn.BatchNorm2d(out_channels)\n        \n        self.use_se = use_se\n        if use_se:\n            self.se = SEBlock(out_channels)\n            \n        self.downsample = None\n        if stride != 1 or in_channels != out_channels:\n            self.downsample = nn.Sequential(\n                nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n                nn.BatchNorm2d(out_channels)\n            )\n\n    def forward(self, x):\n        identity = x\n        \n        out = self.conv1(x)\n        out = self.bn1(out)\n        out = self.relu(out)\n        \n        out = self.conv2(out)\n        out = self.bn2(out)\n        \n        if self.use_se:\n            out = self.se(out)\n            \n        if self.downsample is not None:\n            identity = self.downsample(x)\n            \n        out += identity\n        out = self.relu(out)\n        return out\n\nclass BiomassResNet(nn.Module):\n    def __init__(self, tabular_dim):\n        super().__init__()\n        \n        # Stem (Input -> 64 channels)\n        self.stem = nn.Sequential(\n            nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False),\n            nn.BatchNorm2d(64),\n            nn.ReLU(inplace=True),\n            nn.MaxPool2d(kernel_size=3, stride=2, padding=1)\n        )\n        \n        # ResNet Stages (ResNet-18 style: 2 blocks per stage)\n        self.layer1 = self._make_layer(64, 64, 2, stride=1)\n        self.layer2 = self._make_layer(64, 128, 2, stride=2)\n        self.layer3 = self._make_layer(128, 256, 2, stride=2)\n        self.layer4 = self._make_layer(256, 512, 2, stride=2)\n        \n        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n        \n        # Tabular Fusion\n        self.tab_embed = nn.Sequential(\n            nn.Linear(tabular_dim, 64),\n            nn.BatchNorm1d(64),\n            nn.ReLU(),\n            nn.Linear(64, 64),\n            nn.ReLU()\n        )\n        \n        # Final Head\n        self.head = nn.Sequential(\n            nn.Dropout(0.3),\n            nn.Linear(512 + 64, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(0.3),\n            nn.Linear(256, len(TARGETS))\n        )\n        \n        self.loss_fn = nn.SmoothL1Loss(reduction='none')\n\n    def _make_layer(self, in_c, out_c, blocks, stride):\n        layers = []\n        layers.append(ResBlock(in_c, out_c, stride))\n        for _ in range(1, blocks):\n            layers.append(ResBlock(out_c, out_c))\n        return nn.Sequential(*layers)\n\n    def forward(self, img, tab):\n        # Image path\n        x = self.stem(img)      # 64\n        x = self.layer1(x)      # 64\n        x = self.layer2(x)      # 128\n        x = self.layer3(x)      # 256\n        x = self.layer4(x)      # 512\n        x = self.avgpool(x)\n        x = torch.flatten(x, 1) # [B, 512]\n        \n        # Tabular path\n        t = self.tab_embed(tab) # [B, 64]\n        \n        # Concatenate\n        combined = torch.cat([x, t], dim=1) # [B, 576]\n        return self.head(combined)\n\n    def weighted_loss(self, pred, target, device):\n        weights = TARGET_WEIGHTS.to(device)\n        loss = self.loss_fn(pred, target)\n        return (loss * weights).mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T10:10:01.957910Z","iopub.execute_input":"2025-11-24T10:10:01.958242Z","iopub.status.idle":"2025-11-24T10:10:02.048534Z","shell.execute_reply.started":"2025-11-24T10:10:01.958216Z","shell.execute_reply":"2025-11-24T10:10:02.047884Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from tqdm.auto import tqdm\n\n# Settings\nN_FOLDS = 5\nN_EPOCHS = 40  # Slightly fewer epochs needed with ResNet convergence\nLR = 5e-4      # Lower LR for ResNet from scratch\nDEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nMIXUP_ALPHA = 0.4  # Strength of mixup (0.4 is standard for regression)\n\nkf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\noof_preds = np.zeros((len(train_meta), len(TARGETS)))\ntest_preds_accum = np.zeros((len(test_meta), len(TARGETS)))\n\n# --- Mixup Implementation ---\ndef mixup_data(x, t, y, alpha=0.4):\n    if alpha > 0:\n        lam = np.random.beta(alpha, alpha)\n    else:\n        lam = 1\n    batch_size = x.size()[0]\n    index = torch.randperm(batch_size).to(x.device)\n    \n    mixed_x = lam * x + (1 - lam) * x[index]\n    mixed_t = lam * t + (1 - lam) * t[index] # Mix tabular too\n    y_a, y_b = y, y[index]\n    return mixed_x, mixed_t, y_a, y_b, lam\n\ndef mixup_criterion(criterion, pred, y_a, y_b, lam, device):\n    weights = TARGET_WEIGHTS.to(device)\n    loss_a = (criterion(pred, y_a) * weights).mean()\n    loss_b = (criterion(pred, y_b) * weights).mean()\n    return lam * loss_a + (1 - lam) * loss_b\n\n# Helper for weighted R2\ndef get_weighted_r2(true_log, pred_log):\n    r2_sum = 0\n    weights = [0.1, 0.1, 0.1, 0.2, 0.5]\n    for i, w in enumerate(weights):\n        r2 = r2_score(true_log[:, i], pred_log[:, i])\n        r2_sum += w * r2\n    return r2_sum\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(train_meta)):\n    print(f\"\\n{'='*40}\\nFOLD {fold+1}/{N_FOLDS}\\n{'='*40}\")\n    \n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    # Datasets\n    train_ds = BiomassDataset(train_meta.iloc[train_idx], X_train_tab[train_idx], \n                              is_train=True, targets=Y_train_log[train_idx], \n                              transforms=get_train_transforms(IMG_SIZE))\n    val_ds = BiomassDataset(train_meta.iloc[val_idx], X_train_tab[val_idx], \n                            is_train=True, targets=Y_train_log[val_idx], \n                            transforms=get_test_transforms(IMG_SIZE))\n    test_ds = BiomassDataset(test_meta, X_test_tab, is_train=False, targets=None, \n                             transforms=get_test_transforms(IMG_SIZE))\n    \n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=2, pin_memory=True)\n    val_loader = DataLoader(val_ds, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=2, pin_memory=True)\n    test_loader = DataLoader(test_ds, batch_size=BATCH_SIZE*2, shuffle=False, num_workers=2, pin_memory=True)\n    \n    # Initialize Model\n    model = BiomassResNet(tabular_dim=X_train_tab.shape[1]).to(DEVICE)\n    \n    optimizer = optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-3) # Higher weight decay for ResNet\n    scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=N_EPOCHS, eta_min=1e-6)\n    \n    best_val_r2 = -float('inf')\n    MODEL_PATH = f'best_model_fold_{fold}.pth'\n    \n    for epoch in range(1, N_EPOCHS + 1):\n        model.train()\n        train_losses = []\n        \n        # Training Loop\n        for img, tab, target in tqdm(train_loader, desc=f\"Ep {epoch}\", leave=False):\n            img, tab, target = img.to(DEVICE), tab.to(DEVICE), target.to(DEVICE)\n            \n            # Apply Mixup\n            inputs, tabs, targets_a, targets_b, lam = mixup_data(img, tab, target, MIXUP_ALPHA)\n            \n            optimizer.zero_grad()\n            \n            with torch.cuda.amp.autocast():\n                pred = model(inputs, tabs)\n                loss = mixup_criterion(model.loss_fn, pred, targets_a, targets_b, lam, DEVICE)\n            \n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n            optimizer.step()\n            train_losses.append(loss.item())\n            \n        scheduler.step()\n        \n        # Validation Loop\n        model.eval()\n        val_preds = []\n        val_targets = []\n        \n        with torch.no_grad():\n            for img, tab, target in val_loader:\n                img, tab, target = img.to(DEVICE), tab.to(DEVICE), target.to(DEVICE)\n                with torch.cuda.amp.autocast():\n                    pred = model(img, tab)\n                val_preds.append(pred.cpu().numpy())\n                val_targets.append(target.cpu().numpy())\n                \n        val_preds = np.vstack(val_preds)\n        val_targets = np.vstack(val_targets)\n        \n        # Metric\n        val_r2 = get_weighted_r2(val_targets, val_preds)\n        \n        if val_r2 > best_val_r2:\n            best_val_r2 = val_r2\n            torch.save(model.state_dict(), MODEL_PATH)\n            print(f\"Epoch {epoch}: Train Loss {np.mean(train_losses):.4f} | Val R2 {val_r2:.4f} [SAVED]\")\n        else:\n            print(f\"Epoch {epoch}: Train Loss {np.mean(train_losses):.4f} | Val R2 {val_r2:.4f}\")\n            \n    # Load Best for Inference\n    model.load_state_dict(torch.load(MODEL_PATH))\n    model.eval()\n    \n    # OOF\n    val_preds_final = []\n    with torch.no_grad():\n        for img, tab, _ in val_loader:\n            img, tab = img.to(DEVICE), tab.to(DEVICE)\n            val_preds_final.append(model(img, tab).cpu().numpy())\n    oof_preds[val_idx] = np.vstack(val_preds_final)\n    \n    # Test\n    test_preds_fold = []\n    with torch.no_grad():\n        for img, tab in test_loader:\n            img, tab = img.to(DEVICE), tab.to(DEVICE)\n            test_preds_fold.append(model(img, tab).cpu().numpy())\n    test_preds_accum += np.vstack(test_preds_fold)\n\n    print(f\"Fold {fold+1} Best Weighted R2: {best_val_r2:.4f}\")\n\n# Final Stats\nfinal_test_preds_log = test_preds_accum / N_FOLDS\ntotal_r2 = get_weighted_r2(Y_train_log, oof_preds)\nprint(f\"\\n*** OVERALL OOF R2: {total_r2:.4f} ***\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T10:10:02.049473Z","iopub.execute_input":"2025-11-24T10:10:02.049808Z","iopub.status.idle":"2025-11-24T11:32:15.297389Z","shell.execute_reply.started":"2025-11-24T10:10:02.049763Z","shell.execute_reply":"2025-11-24T11:32:15.296368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CELL 5 – Generate Submission\nprint(\"\\nGenerating submission...\")\nprint(f\"Prediction stats (log scale): min={final_test_preds_log.min():.2f}, max={final_test_preds_log.max():.2f}\")\n\n# Clip negative log predictions to 0\nfinal_test_preds_log = np.maximum(0, final_test_preds_log)\n\n# Inverse transform\nfinal_test_preds_original = np.expm1(final_test_preds_log)\n\n# Create lookup\ntest_meta_pred = test_meta.copy()\nfor i, t in enumerate(TARGETS):\n    test_meta_pred[f'pred_{t}'] = final_test_preds_original[:, i]\n\n# Build submission\nsubmission_rows = []\nfor _, row in test_df.iterrows():\n    image_id = row['image_id']\n    target_name = row['target_name']\n    sample_id = row['sample_id']\n    \n    pred_col = f'pred_{target_name}'\n    pred_value = test_meta_pred[test_meta_pred['image_id'] == image_id][pred_col].values[0]\n    \n    submission_rows.append({'sample_id': sample_id, 'target': float(pred_value)})\n\nsubmission = pd.DataFrame(submission_rows)\nsubmission.to_csv('submission.csv', index=False)\n\nprint(f\"\\n✅ Submission created with {len(submission)} rows\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T11:32:15.572891Z","iopub.execute_input":"2025-11-24T11:32:15.573471Z","iopub.status.idle":"2025-11-24T11:32:15.596459Z","shell.execute_reply.started":"2025-11-24T11:32:15.573449Z","shell.execute_reply":"2025-11-24T11:32:15.595755Z"}},"outputs":[],"execution_count":null}]}