{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":112509,"databundleVersionId":14254895}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Full improved pipeline - copy-paste into a notebook cell\n# NOTE: This script is intended to be run in a Kaggle-style environment\n# Adjust image paths if necessary.\n\nimport os\nimport random\nimport math\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageFile\nImageFile.LOAD_TRUNCATED_IMAGES = True\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nimport torch.optim as optim\n\nfrom sklearn.model_selection import train_test_split, KFold\nfrom sklearn.linear_model import Ridge\nfrom sklearn.metrics import mean_squared_error, mean_absolute_error\nimport xgboost as xgb\n\n# --------------------------\n# Reproducibility / device\n# --------------------------\nSEED = 42\nrandom.seed(SEED)\nnp.random.seed(SEED)\ntorch.manual_seed(SEED)\ntorch.cuda.manual_seed_all(SEED)\ntorch.backends.cudnn.deterministic = True\ntorch.backends.cudnn.benchmark = False\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(\"Device:\", device)\n\n# --------------------------\n# Paths / Basic configs\n# --------------------------\nTRAIN_CSV = '/kaggle/input/csiro-biomass/train.csv'\nTEST_CSV  = '/kaggle/input/csiro-biomass/test.csv'\nTRAIN_IMG_DIR = '/kaggle/input/csiro-biomass/train'  # images folder\nTEST_IMG_DIR  = '/kaggle/input/csiro-biomass/test'\n\nN_FOLDS = 5\nBATCH_SIZE = 32\nNUM_WORKERS = 4\nIMAGE_SIZE = 224  # can increase if GPU allows\nNUM_EPOCHS_HEAD = 3   # epochs training head only\nNUM_EPOCHS_FINE = 7   # epochs after unfreeze\nLR_HEAD = 1e-3\nLR_FINE = 1e-4\n\n# --------------------------\n# Load CSVs and build canonical train_final\n# --------------------------\ntrain_df = pd.read_csv(TRAIN_CSV)\ntest_df  = pd.read_csv(TEST_CSV)\n\n# pivot long->wide: one row per image\ndf_wide = train_df.pivot(index='image_path', columns='target_name', values='target').reset_index()\n\n# bring metadata (choose columns present in your CSV)\nmeta_cols = [c for c in ['image_path','Sampling_Date','State','Species','Pre_GSHH_NDVI','Height_Ave_cm'] if c in train_df.columns]\nif len(meta_cols) > 1:\n    meta = train_df[meta_cols].drop_duplicates(subset='image_path')\n    train_final = pd.merge(df_wide, meta, on='image_path', how='left')\nelse:\n    train_final = df_wide.copy()\n\n# target columns (ensure they exist)\ntarget_cols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nfor t in target_cols:\n    if t not in train_final.columns:\n        raise ValueError(f\"Required target column {t} not present in pivoted train_final.\")\n\n# drop rows with missing targets\ntrain_final = train_final.dropna(subset=target_cols).reset_index(drop=True)\nprint(\"train_final rows after dropna:\", len(train_final))\n\n# apply log1p transform to stabilize skew; keep original values if needed\nfor c in target_cols:\n    train_final[c + '_log'] = np.log1p(train_final[c].astype(float))\n\n# --------------------------\n# Transforms and Dataset\n# --------------------------\ntrain_transform = transforms.Compose([\n    transforms.RandomResizedCrop(IMAGE_SIZE, scale=(0.8,1.0)),\n    transforms.RandomHorizontalFlip(),\n    transforms.RandomRotation(10),\n    transforms.ColorJitter(0.1,0.1,0.1,0.05),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225]),\n])\n\nval_transform = transforms.Compose([\n    transforms.Resize((IMAGE_SIZE,IMAGE_SIZE)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225]),\n])\n\n# TTA transforms (example: original + horizontal flip)\ntta_transforms = [\n    val_transform,\n    transforms.Compose([transforms.Resize((IMAGE_SIZE,IMAGE_SIZE)),\n                        transforms.RandomHorizontalFlip(p=1.0),\n                        transforms.ToTensor(),\n                        transforms.Normalize(mean=[0.485,0.456,0.406], std=[0.229,0.224,0.225])])\n]\n\nclass BiomassDataset(Dataset):\n    def __init__(self, df, image_dir, transform=None, targets_col=None, is_test=False, return_index=False):\n        self.df = df.reset_index(drop=True)\n        self.image_dir = image_dir\n        self.transform = transform\n        self.targets_col = targets_col\n        self.is_test = is_test\n        self.return_index = return_index\n\n    def __len__(self):\n        return len(self.df)\n\n    def _load_image(self, p):\n        # defensive image loading\n        img_name = str(p).split('/')[-1]\n        path = os.path.join(self.image_dir, img_name)\n        if not os.path.exists(path):\n            # If missing, create a blank image to avoid runtime error (log a warning)\n            # In production you'd rather raise error or skip\n            print(f\"Warning: missing image {path}. Using blank image.\")\n            img = Image.new('RGB', (IMAGE_SIZE, IMAGE_SIZE), color=(128,128,128))\n        else:\n            img = Image.open(path).convert('RGB')\n        return img\n\n    def __getitem__(self, idx):\n        row = self.df.loc[idx]\n        img = self._load_image(row['image_path'])\n        if self.transform:\n            img = self.transform(img)\n        if self.is_test:\n            if self.return_index:\n                return img, idx\n            return img\n        targets = row[self.targets_col].astype(np.float32)\n        if self.return_index:\n            return img, torch.tensor(targets), idx\n        return img, torch.tensor(targets)\n\n# --------------------------\n# Model helpers: ResNet backbone + embedding extractor\n# --------------------------\ndef get_resnet_backbone(pretrained=True, embedding=True):\n    # Using modern weights API if available\n    try:\n        res = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1 if pretrained else None)\n    except Exception:\n        res = models.resnet18(pretrained=pretrained)\n    if embedding:\n        # return model that outputs features instead of final logits\n        res.fc = nn.Identity()\n        feat_dim = res.fc.in_features if False else 512  # resnet18 final feature size is 512\n        return res, feat_dim\n    else:\n        num_features = res.fc.in_features\n        # replace fc with linear head for multi-target regression\n        res.fc = nn.Linear(num_features, len(target_cols))\n        return res, res.fc.in_features\n\n# --------------------------\n# Smooth L1 metric (numpy) for evaluation\n# --------------------------\ndef smooth_l1_numpy(y_true, y_pred, beta=1.0):\n    diff = np.abs(y_true - y_pred)\n    loss = np.where(diff < beta, 0.5 * (diff ** 2) / beta, diff - 0.5 * beta)\n    return np.mean(loss)\n\n# --------------------------\n# K-Fold loop to build OOF predictions (CNN embeddings + CNN preds)\n# --------------------------\nkf = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED)\n\nn_samples = len(train_final)\noof_cnn_preds = np.zeros((n_samples, len(target_cols)), dtype=np.float32)   # will hold cnn direct preds (log-space)\noof_cnn_embs  = np.zeros((n_samples, 512), dtype=np.float32)  # resnet18 embedding dim 512\noof_xgb_preds = np.zeros((n_samples, len(target_cols)), dtype=np.float32)   # xgb predictions (log-space), OOF\n\ntest_cnn_preds_folds = []  # collect test preds per fold to average later (for direct cnn predictions)\ntest_cnn_embs_folds  = []  # collect test embeddings to average for XGB test training\n\n# create test dataset loader for embedding extraction later (TTA not used for emb extraction)\ntest_ds_for_emb = BiomassDataset(test_df, TEST_IMG_DIR, transform=val_transform, is_test=True, return_index=False)\ntest_loader_for_emb = DataLoader(test_ds_for_emb, batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS)\n\nfold_idx = 0\nfor train_idx, val_idx in kf.split(train_final):\n    fold_idx += 1\n    print(f\"\\n=== Fold {fold_idx}/{N_FOLDS} ===\")\n    train_fold = train_final.iloc[train_idx].reset_index(drop=True)\n    val_fold   = train_final.iloc[val_idx].reset_index(drop=True)\n\n    # Datasets and loaders (use label columns = log targets)\n    tgt_cols_log = [c + '_log' for c in target_cols]\n    train_ds = BiomassDataset(train_fold, TRAIN_IMG_DIR, transform=train_transform, targets_col=tgt_cols_log)\n    val_ds   = BiomassDataset(val_fold, TRAIN_IMG_DIR, transform=val_transform, targets_col=tgt_cols_log)\n    train_loader = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, num_workers=NUM_WORKERS, pin_memory=True)\n    val_loader   = DataLoader(val_ds,   batch_size=BATCH_SIZE, shuffle=False, num_workers=NUM_WORKERS, pin_memory=True)\n\n    # Build model: embedding backbone and separate regression head\n    backbone, feat_dim = get_resnet_backbone(pretrained=True, embedding=True)\n    # head: small MLP to predict log-targets\n    class RegressionHead(nn.Module):\n        def __init__(self, feat_dim, num_targets):\n            super().__init__()\n            self.fc = nn.Sequential(\n                nn.Linear(feat_dim, 512),\n                nn.ReLU(),\n                nn.Dropout(0.3),\n                nn.Linear(512, num_targets)\n            )\n        def forward(self, x):\n            return self.fc(x)\n\n    head = RegressionHead(feat_dim, len(target_cols))\n    model = nn.Sequential(backbone, head)  # outputs (batch, num_targets)\n    model = model.to(device)\n\n    # freeze backbone initially (train head only)\n    for p in backbone.parameters():\n        p.requires_grad = False\n    for p in head.parameters():\n        p.requires_grad = True\n\n    optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=LR_HEAD, weight_decay=1e-4)\n    criterion = nn.SmoothL1Loss()  # PyTorch version (uses beta=1.0 default semantics)\n    scaler = torch.cuda.amp.GradScaler() if device.type == 'cuda' else None\n\n    # Training: head-only\n    for epoch in range(NUM_EPOCHS_HEAD):\n        model.train()\n        total_loss = 0.0\n        for imgs, targets in train_loader:\n            imgs = imgs.to(device)\n            targets = targets.to(device)\n            optimizer.zero_grad()\n            if scaler:\n                with torch.cuda.amp.autocast():\n                    outputs = model(imgs)\n                    loss = criterion(outputs, targets)\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                outputs = model(imgs)\n                loss = criterion(outputs, targets)\n                loss.backward()\n                optimizer.step()\n            total_loss += loss.item() * imgs.size(0)\n        avg_loss = total_loss / len(train_loader.dataset)\n        print(f\"Fold {fold_idx} Head Epoch {epoch+1}/{NUM_EPOCHS_HEAD} - loss: {avg_loss:.4f}\")\n\n    # Unfreeze last layers of backbone (layer3, layer4) for fine-tuning\n    for name, p in backbone.named_parameters():\n        if 'layer3' in name or 'layer4' in name:\n            p.requires_grad = True\n    optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()), lr=LR_FINE, weight_decay=1e-5)\n    # LR scheduler optional:\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=3, factor=0.5, verbose=True)\n\n    # Fine-tuning epochs\n    for epoch in range(NUM_EPOCHS_FINE):\n        model.train()\n        total_loss = 0.0\n        for imgs, targets in train_loader:\n            imgs = imgs.to(device)\n            targets = targets.to(device)\n            optimizer.zero_grad()\n            if scaler:\n                with torch.cuda.amp.autocast():\n                    outputs = model(imgs)\n                    loss = criterion(outputs, targets)\n                scaler.scale(loss).backward()\n                scaler.step(optimizer)\n                scaler.update()\n            else:\n                outputs = model(imgs)\n                loss = criterion(outputs, targets)\n                loss.backward()\n                optimizer.step()\n            total_loss += loss.item() * imgs.size(0)\n        avg_loss = total_loss / len(train_loader.dataset)\n        # validation for scheduler\n        # compute val loss (no grads)\n        model.eval()\n        val_loss = 0.0\n        with torch.no_grad():\n            for imgs, targets in val_loader:\n                imgs = imgs.to(device)\n                targets = targets.to(device)\n                if scaler:\n                    with torch.cuda.amp.autocast():\n                        outputs = model(imgs)\n                        loss = criterion(outputs, targets)\n                else:\n                    outputs = model(imgs)\n                    loss = criterion(outputs, targets)\n                val_loss += loss.item() * imgs.size(0)\n        val_loss = val_loss / len(val_loader.dataset)\n        scheduler.step(val_loss)\n        print(f\"Fold {fold_idx} Fine Epoch {epoch+1}/{NUM_EPOCHS_FINE} - train_loss: {avg_loss:.4f}, val_loss: {val_loss:.4f}\")\n\n    # After training: extract embeddings and predictions for val_fold (aligned) and test\n    # Create embedding-only model: backbone (already producing embeddings)\n    backbone.eval()\n    head.eval()\n    # VAL embeddings & preds\n    all_val_embs = []\n    all_val_preds = []\n    with torch.no_grad():\n        for imgs, _ in val_loader:\n            imgs = imgs.to(device)\n            emb = backbone(imgs)  # (bs, feat_dim)\n            out = head(emb)       # (bs, num_targets) - log space\n            all_val_embs.append(emb.cpu().numpy())\n            all_val_preds.append(out.cpu().numpy())\n    all_val_embs = np.vstack(all_val_embs)\n    all_val_preds = np.vstack(all_val_preds)\n\n    # write into OOF arrays at val_idx positions\n    oof_cnn_preds[val_idx] = all_val_preds\n    oof_cnn_embs[val_idx]  = all_val_embs\n\n    # TEST: get embeddings and preds for test set and store per-fold\n    test_embs_fold = []\n    test_preds_fold = []\n    with torch.no_grad():\n        for imgs in test_loader_for_emb:\n            imgs = imgs.to(device)\n            emb = backbone(imgs)\n            out = head(emb)\n            test_embs_fold.append(emb.cpu().numpy())\n            test_preds_fold.append(out.cpu().numpy())\n    test_embs_fold = np.vstack(test_embs_fold)\n    test_preds_fold = np.vstack(test_preds_fold)\n\n    test_cnn_embs_folds.append(test_embs_fold)\n    test_cnn_preds_folds.append(test_preds_fold)\n\n    fold_gc = torch.cuda.empty_cache if device.type == 'cuda' else lambda : None\n    fold_gc()\n    print(f\"Fold {fold_idx} completed. Val preds shape: {all_val_preds.shape}, Val embs shape: {all_val_embs.shape}\")\n\n# --------------------------\n# Average test embeddings/preds across folds\n# --------------------------\ntest_cnn_embs_avg = np.mean(np.stack(test_cnn_embs_folds, axis=0), axis=0)  # (num_test, emb_dim)\ntest_cnn_preds_avg = np.mean(np.stack(test_cnn_preds_folds, axis=0), axis=0)  # (num_test, num_targets)\n\nprint(\"OOF CNN preds shape:\", oof_cnn_preds.shape)\nprint(\"OOF CNN embs shape:\", oof_cnn_embs.shape)\nprint(\"Test CNN embs avg shape:\", test_cnn_embs_avg.shape)\n\n# --------------------------\n# Train XGBoost per target using CNN embeddings as features (OOF style)\n# --------------------------\nxgb_models = []\noof_xgb_preds = np.zeros_like(oof_cnn_preds)\ntest_xgb_preds_folds = []\n\n# We'll train XGB on embeddings from the training folds (use same OOF split as above)\nfor i, t in enumerate(target_cols):\n    print(f\"Training XGB for target {t} (log-space)...\")\n    y_train_log = train_final[t + '_log'].values  # aligned to train_final order\n\n    params = {\n        'n_estimators': 200,\n        'max_depth': 6,\n        'learning_rate': 0.05,\n        'subsample': 0.8,\n        'colsample_bytree': 0.8,\n        'random_state': SEED,\n        'n_jobs': -1,\n        'verbosity': 0\n    }\n    model = xgb.XGBRegressor(**params)\n    # fit on full training embeddings (but better: use OOF scheme). We'll do KFold OOF for XGB:\n    oof_preds_target = np.zeros(n_samples)\n    test_preds_folds = []\n\n    kf2 = KFold(n_splits=N_FOLDS, shuffle=True, random_state=SEED+1)\n    for tr_idx, vl_idx in kf2.split(oof_cnn_embs):\n        X_tr = oof_cnn_embs[tr_idx]\n        y_tr = y_train_log[tr_idx]\n        X_vl = oof_cnn_embs[vl_idx]\n\n        m = xgb.XGBRegressor(**params)\n        m.fit(X_tr, y_tr, eval_set=[(X_vl, y_train_log[vl_idx])], early_stopping_rounds=30, verbose=False)\n        oof_preds_target[vl_idx] = m.predict(X_vl)\n        test_preds_folds.append(m.predict(test_cnn_embs_avg))\n    oof_xgb_preds[:, i] = oof_preds_target\n    test_xgb_preds_folds.append(np.mean(np.stack(test_preds_folds, axis=0), axis=0))\n    xgb_models.append(m)  # last trained model (not used further)\n\ntest_xgb_preds_avg = np.column_stack(test_xgb_preds_folds)  # (num_test, num_targets)\nprint(\"XGB OOF preds shape:\", oof_xgb_preds.shape)\nprint(\"XGB test preds shape:\", test_xgb_preds_avg.shape)\n\n# --------------------------\n# Meta-features & meta-model training (Ridge)\n#   We will use for each sample: [cnn_pred_i, xgb_pred_i] for each target\n# --------------------------\n# Combine OOF features\nmeta_oof_features = np.concatenate([oof_cnn_preds, oof_xgb_preds], axis=1)  # (n_samples, 10)\nmeta_test_features  = np.concatenate([test_cnn_preds_avg, test_xgb_preds_avg], axis=1)  # (num_test, 10)\n\nmeta_models = []\nmeta_test_preds = np.zeros((meta_test_features.shape[0], len(target_cols)))\n\nfor i, t in enumerate(target_cols):\n    # features: column i (cnn), column i+5 (xgb)\n    X_meta = meta_oof_features[:, [i, i + len(target_cols)]]\n    y_meta = train_final[t + '_log'].values\n    m = Ridge(alpha=1.0)\n    m.fit(X_meta, y_meta)\n    meta_models.append(m)\n    meta_test_preds[:, i] = m.predict(meta_test_features[:, [i, i + len(target_cols)]])\n\n# --------------------------\n# Invert log transform for predictions\n# --------------------------\n# For evaluation on validation set we need to compare to real targets.\n# We have OOF stacked preds (on log scale). Let's compute OOF stacked preds:\nstack_oof_preds_log = np.zeros_like(oof_cnn_preds)\nfor i, m in enumerate(meta_models):\n    X_meta = meta_oof_features[:, [i, i + len(target_cols)]]\n    stack_oof_preds_log[:, i] = m.predict(X_meta)\n\nstack_oof_preds = np.expm1(stack_oof_preds_log)  # back to original target scale\ny_true = train_final[target_cols].values  # original scale\n\n# --------------------------\n# Metrics: MSE, MAE, SmoothL1\n# --------------------------\nmse_list, mae_list, smooth1_list = [], [], []\nprint(\"\\nValidation metrics (OOF):\")\nfor i, t in enumerate(target_cols):\n    mse = mean_squared_error(y_true[:, i], stack_oof_preds[:, i])\n    mae = mean_absolute_error(y_true[:, i], stack_oof_preds[:, i])\n    s1 = smooth_l1_numpy(y_true[:, i], stack_oof_preds[:, i], beta=1.0)\n    print(f\"{t} - MSE: {mse:.4f}, MAE: {mae:.4f}, SmoothL1: {s1:.4f}\")\n    mse_list.append(mse); mae_list.append(mae); smooth1_list.append(s1)\n\nprint(f\"\\nMean MSE: {np.mean(mse_list):.4f}, Mean MAE: {np.mean(mae_list):.4f}, Mean SmoothL1: {np.mean(smooth1_list):.4f}\")\n\n# --------------------------\n# Final test predictions (meta_test_preds are in log-space)\n# --------------------------\nfinal_test_preds_log = meta_test_preds\nfinal_test_preds = np.expm1(final_test_preds_log)  # invert\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-07T09:46:06.634756Z","iopub.execute_input":"2025-11-07T09:46:06.635519Z","iopub.status.idle":"2025-11-07T09:56:43.400508Z","shell.execute_reply.started":"2025-11-07T09:46:06.635487Z","shell.execute_reply":"2025-11-07T09:56:43.399719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}