{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13821154,"sourceType":"datasetVersion","datasetId":8801749},{"sourceId":4534,"sourceType":"modelInstanceVersion","modelInstanceId":3326,"modelId":986},{"sourceId":4535,"sourceType":"modelInstanceVersion","modelInstanceId":3327,"modelId":986},{"sourceId":4537,"sourceType":"modelInstanceVersion","modelInstanceId":3329,"modelId":986},{"sourceId":6119,"sourceType":"modelInstanceVersion","modelInstanceId":4602,"modelId":2797},{"sourceId":268942,"sourceType":"modelInstanceVersion","modelInstanceId":230141,"modelId":251887},{"sourceId":646733,"sourceType":"modelInstanceVersion","modelInstanceId":487751,"modelId":503172},{"sourceId":658655,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":497995,"modelId":513289}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# \"\"\"\n# ULTIMATE ENSEMBLE - ONLY AVAILABLE MODELS\n# ✅ Uses only: DINOv2 (base/large/giant) + SigLIP\n# ✅ No ConvNeXt (not available)\n# ✅ Same aggressive tuning & working submission format\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n# from pathlib import Path\n# from collections import OrderedDict\n# from scipy.optimize import differential_evolution\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n\n# from PIL import Image\n# import cv2\n\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import KFold\n# from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor\n# from sklearn.metrics import r2_score\n\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n# import timm\n\n# import lightgbm as lgb\n# import xgboost as xgb\n# import optuna\n# from optuna.samplers import TPESampler\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 Ultimate Ensemble - Available Models Only\")\n# print(f\"💻 GPU Count: {torch.cuda.device_count()}\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION - AGGRESSIVE\n# # ============================================================================\n\n# class Config:\n#     epochs = 20  # 25 epochs fine-tuning\n#     batch_size = 8\n#     lr = 1e-4\n#     weight_decay = 0.01\n#     img_size = 384\n#     n_folds = 3\n#     n_optuna_trials = 50  # Aggressive tuning\n#     seed = 42\n#     num_workers = 4\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # DATA LOADING\n# # ============================================================================\n\n# def load_data():\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     # Pivot to get one row per image\n#     train_pivot = train_df.pivot_table(\n#         index='image_path', columns='target_name', values='target', aggfunc='first'\n#     ).reset_index()\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # MODEL 1: FINE-TUNE DINOv2 GIANT (25 EPOCHS)\n# # ============================================================================\n\n# class DINOv2GiantFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Freeze most layers, only train last few\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1536, 768),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(768, 384),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(384, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.processor = AutoImageProcessor.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img = Image.open(os.path.join(ROOT, self.image_paths[idx]))\n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# def train_dinov2_giant(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2-GIANT (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2GiantFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_features = []\n#     all_preds = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx])\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size, \n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size, \n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_loss = float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation\n#             model.eval()\n#             val_loss = 0\n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     val_loss += criterion(outputs, batch_y).item()\n            \n#             val_loss /= len(val_loader)\n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n            \n#             if val_loss < best_val_loss:\n#                 best_val_loss = val_loss\n#                 torch.save(model.state_dict(), f'dinov2_giant_fold{fold}.pth')\n        \n#         # Extract features for this fold\n#         model.load_state_dict(torch.load(f'dinov2_giant_fold{fold}.pth'))\n#         model.eval()\n        \n#         with torch.no_grad():\n#             for batch_x in DataLoader(DINOv2Dataset(X[val_idx]), batch_size=config.batch_size):\n#                 batch_x = batch_x.to(DEVICE)\n#                 features = model.backbone(pixel_values=batch_x).pooler_output\n#                 all_features.append(features.cpu().numpy())\n#                 preds = model(batch_x)\n#                 all_preds.append(preds.cpu().numpy())\n    \n#     print(f\"\\n✓ DINOv2 Giant training complete\")\n#     return model, np.vstack(all_features), np.vstack(all_preds)\n\n# # ============================================================================\n# # MODEL 2: FINE-TUNE DINOv2 LARGE (25 EPOCHS)\n# # ============================================================================\n\n# class DINOv2LargeFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/large/1')\n        \n#         # Freeze most layers, only train last few\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1024, 512),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 256),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(256, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# def train_dinov2_large(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2-LARGE (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2LargeFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_features = []\n#     all_preds = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx])\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size, \n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size, \n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_loss = float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation\n#             model.eval()\n#             val_loss = 0\n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     val_loss += criterion(outputs, batch_y).item()\n            \n#             val_loss /= len(val_loader)\n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n            \n#             if val_loss < best_val_loss:\n#                 best_val_loss = val_loss\n#                 torch.save(model.state_dict(), f'dinov2_large_fold{fold}.pth')\n        \n#         # Extract features for this fold\n#         model.load_state_dict(torch.load(f'dinov2_large_fold{fold}.pth'))\n#         model.eval()\n        \n#         with torch.no_grad():\n#             for batch_x in DataLoader(DINOv2Dataset(X[val_idx]), batch_size=config.batch_size):\n#                 batch_x = batch_x.to(DEVICE)\n#                 features = model.backbone(pixel_values=batch_x).pooler_output\n#                 all_features.append(features.cpu().numpy())\n#                 preds = model(batch_x)\n#                 all_preds.append(preds.cpu().numpy())\n    \n#     print(f\"\\n✓ DINOv2 Large training complete\")\n#     return model, np.vstack(all_features), np.vstack(all_preds)\n\n# # ============================================================================\n# # MODEL 3: FINE-TUNE SigLIP (25 EPOCHS)\n# # ============================================================================\n\n# class SigLIPFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         model_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\n#         self.backbone = AutoModel.from_pretrained(model_path)\n        \n#         # Freeze most layers\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.vision_model.encoder.layers[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1152, 576),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(576, 288),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(288, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone.get_image_features(pixel_values=x)\n#         return self.head(features)\n\n# class SigLIPDataset(Dataset):\n#     def __init__(self, image_paths, targets=None):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         model_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\n#         self.processor = AutoImageProcessor.from_pretrained(model_path)\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img = Image.open(os.path.join(ROOT, self.image_paths[idx]))\n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# def train_siglip(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING SigLIP (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = SigLIPFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_features = []\n#     all_preds = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = SigLIPDataset(X[train_idx], y[train_idx])\n#         val_dataset = SigLIPDataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size,\n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size,\n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_loss = float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation\n#             model.eval()\n#             val_loss = 0\n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     val_loss += criterion(outputs, batch_y).item()\n            \n#             val_loss /= len(val_loader)\n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n            \n#             if val_loss < best_val_loss:\n#                 best_val_loss = val_loss\n#                 torch.save(model.state_dict(), f'siglip_fold{fold}.pth')\n        \n#         # Extract features\n#         model.load_state_dict(torch.load(f'siglip_fold{fold}.pth'))\n#         model.eval()\n        \n#         with torch.no_grad():\n#             for batch_x in DataLoader(SigLIPDataset(X[val_idx]), batch_size=config.batch_size):\n#                 batch_x = batch_x.to(DEVICE)\n#                 features = model.backbone.get_image_features(pixel_values=batch_x)\n#                 all_features.append(features.cpu().numpy())\n#                 preds = model(batch_x)\n#                 all_preds.append(preds.cpu().numpy())\n    \n#     print(f\"\\n✓ SigLIP training complete\")\n#     return model, np.vstack(all_features), np.vstack(all_preds)\n\n# # ============================================================================\n# # AGGRESSIVE HYPERPARAMETER TUNING\n# # ============================================================================\n\n# def tune_and_train_gbms(X_train, y_train, X_val, y_val):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"AGGRESSIVE HYPERPARAMETER TUNING\")\n#     print(\"=\"*80)\n    \n#     models = {}\n#     predictions = {}\n    \n#     for target_idx, target_name in enumerate(labels):\n#         print(f\"\\n📊 Target: {target_name}\")\n        \n#         # LightGBM with aggressive tuning\n#         print(\"  Tuning LightGBM...\")\n#         study_lgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=config.seed))\n        \n#         def objective_lgb(trial):\n#             params = {\n#                 'objective': 'regression',\n#                 'metric': 'rmse',\n#                 'boosting_type': 'gbdt',\n#                 'verbosity': -1,\n#                 'device': 'gpu' if torch.cuda.is_available() else 'cpu',\n#                 'seed': config.seed,\n#                 'num_leaves': trial.suggest_int('num_leaves', 128, 2048),\n#                 'max_depth': trial.suggest_int('max_depth', 6, 20),\n#                 'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.2, log=True),\n#                 'n_estimators': trial.suggest_int('n_estimators', 2000, 15000),\n#                 'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n#                 'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#                 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#                 'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),\n#                 'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),\n#             }\n#             model = lgb.LGBMRegressor(**params)\n#             model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       callbacks=[lgb.early_stopping(300, verbose=False)])\n#             preds = model.predict(X_val)\n#             return r2_score(y_val[:, target_idx], preds)\n        \n#         study_lgb.optimize(objective_lgb, n_trials=config.n_optuna_trials, show_progress_bar=True)\n#         best_params_lgb = study_lgb.best_params\n#         best_params_lgb.update({'objective': 'regression', 'metric': 'rmse', \n#                                 'boosting_type': 'gbdt', 'verbosity': -1,\n#                                 'device': 'gpu' if torch.cuda.is_available() else 'cpu',\n#                                 'seed': config.seed})\n        \n#         lgb_model = lgb.LGBMRegressor(**best_params_lgb)\n#         lgb_model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       callbacks=[lgb.early_stopping(300, verbose=False)])\n#         models[f'lgb_{target_idx}'] = lgb_model\n#         predictions[f'lgb_{target_idx}'] = lgb_model.predict(X_val)\n        \n#         # XGBoost with aggressive tuning\n#         print(\"  Tuning XGBoost...\")\n#         study_xgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=config.seed))\n        \n#         def objective_xgb(trial):\n#             params = {\n#                 'objective': 'reg:squarederror',\n#                 'tree_method': 'gpu_hist' if torch.cuda.is_available() else 'hist',\n#                 'seed': config.seed,\n#                 'max_depth': trial.suggest_int('max_depth', 6, 20),\n#                 'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.2, log=True),\n#                 'n_estimators': trial.suggest_int('n_estimators', 2000, 15000),\n#                 'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n#                 'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#                 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#                 'gamma': trial.suggest_float('gamma', 1e-8, 5.0, log=True),\n#                 'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),\n#                 'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),\n#             }\n#             model = xgb.XGBRegressor(**params)\n#             model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       verbose=False)\n#             preds = model.predict(X_val)\n#             return r2_score(y_val[:, target_idx], preds)\n        \n#         study_xgb.optimize(objective_xgb, n_trials=config.n_optuna_trials, show_progress_bar=True)\n#         best_params_xgb = study_xgb.best_params\n#         best_params_xgb.update({'objective': 'reg:squarederror',\n#                                 'tree_method': 'gpu_hist' if torch.cuda.is_available() else 'hist',\n#                                 'seed': config.seed})\n        \n#         xgb_model = xgb.XGBRegressor(**best_params_xgb)\n#         xgb_model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       verbose=False)\n#         models[f'xgb_{target_idx}'] = xgb_model\n#         predictions[f'xgb_{target_idx}'] = xgb_model.predict(X_val)\n        \n#         # ExtraTrees with tuning\n#         print(\"  Tuning ExtraTrees...\")\n#         et_model = ExtraTreesRegressor(\n#             n_estimators=1500,\n#             max_depth=25,\n#             min_samples_split=2,\n#             min_samples_leaf=1,\n#             max_features='sqrt',\n#             bootstrap=True,\n#             n_jobs=-1,\n#             random_state=config.seed\n#         )\n#         et_model.fit(X_train, y_train[:, target_idx])\n#         models[f'et_{target_idx}'] = et_model\n#         predictions[f'et_{target_idx}'] = et_model.predict(X_val)\n        \n#         # RandomForest with tuning\n#         print(\"  Tuning RandomForest...\")\n#         rf_model = RandomForestRegressor(\n#             n_estimators=1500,\n#             max_depth=25,\n#             min_samples_split=2,\n#             min_samples_leaf=1,\n#             max_features='log2',\n#             bootstrap=True,\n#             n_jobs=-1,\n#             random_state=config.seed\n#         )\n#         rf_model.fit(X_train, y_train[:, target_idx])\n#         models[f'rf_{target_idx}'] = rf_model\n#         predictions[f'rf_{target_idx}'] = rf_model.predict(X_val)\n        \n#         print(f\"  ✓ LGB R²: {r2_score(y_val[:, target_idx], predictions[f'lgb_{target_idx}']):.4f}\")\n#         print(f\"  ✓ XGB R²: {r2_score(y_val[:, target_idx], predictions[f'xgb_{target_idx}']):.4f}\")\n#         print(f\"  ✓ ET R²: {r2_score(y_val[:, target_idx], predictions[f'et_{target_idx}']):.4f}\")\n#         print(f\"  ✓ RF R²: {r2_score(y_val[:, target_idx], predictions[f'rf_{target_idx}']):.4f}\")\n    \n#     return models, predictions\n\n# # ============================================================================\n# # WEIGHT OPTIMIZATION\n# # ============================================================================\n\n# def optimize_ensemble_weights(predictions_dict, y_val):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"OPTIMIZING ENSEMBLE WEIGHTS\")\n#     print(\"=\"*80)\n    \n#     # Organize predictions by target\n#     target_preds = {i: [] for i in range(5)}\n#     for key, pred in predictions_dict.items():\n#         target_idx = int(key.split('_')[-1])\n#         target_preds[target_idx].append(pred)\n    \n#     # Optimize weights for each target\n#     optimal_weights = {}\n    \n#     for target_idx in range(5):\n#         preds_list = target_preds[target_idx]\n#         n_models = len(preds_list)\n        \n#         def objective(weights):\n#             weights = np.abs(weights) / np.abs(weights).sum()\n#             ensemble = sum(w * p for w, p in zip(weights, preds_list))\n#             return -r2_score(y_val[:, target_idx], ensemble)\n        \n#         result = differential_evolution(\n#             objective,\n#             bounds=[(0, 1)] * n_models,\n#             seed=config.seed,\n#             maxiter=200,\n#             workers=1\n#         )\n        \n#         optimal_weights[target_idx] = np.abs(result.x) / np.abs(result.x).sum()\n        \n#         print(f\"  {labels[target_idx]}: R² = {-result.fun:.4f}\")\n#         print(f\"    Weights: LGB={optimal_weights[target_idx][0]:.3f}, \"\n#               f\"XGB={optimal_weights[target_idx][1]:.3f}, \"\n#               f\"ET={optimal_weights[target_idx][2]:.3f}, \"\n#               f\"RF={optimal_weights[target_idx][3]:.3f}\")\n    \n#     return optimal_weights\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS\n# # ============================================================================\n\n# def generate_test_predictions(dinov2_giant_model, dinov2_large_model, siglip_model, \n#                                gbm_models, optimal_weights, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df.drop_duplicates('image_path').reset_index(drop=True)\n#     X_test = test_unique['image_path'].values\n    \n#     # Extract features from all 3 models\n#     print(\"\\n1. Extracting DINOv2 Giant features...\")\n#     dinov2_giant_model.eval()\n#     dinov2_giant_features = []\n#     dinov2_giant_preds = []\n    \n#     test_dataset = DINOv2Dataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size, \n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             features = dinov2_giant_model.backbone(pixel_values=batch_x).pooler_output\n#             dinov2_giant_features.append(features.cpu().numpy())\n#             preds = dinov2_giant_model(batch_x)\n#             dinov2_giant_preds.append(preds.cpu().numpy())\n    \n#     dinov2_giant_features = np.vstack(dinov2_giant_features)\n#     dinov2_giant_preds = np.vstack(dinov2_giant_preds)\n    \n#     # DINOv2 Large features\n#     print(\"\\n2. Extracting DINOv2 Large features...\")\n#     dinov2_large_model.eval()\n#     dinov2_large_features = []\n#     dinov2_large_preds = []\n    \n#     test_dataset = DINOv2Dataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size, \n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             features = dinov2_large_model.backbone(pixel_values=batch_x).pooler_output\n#             dinov2_large_features.append(features.cpu().numpy())\n#             preds = dinov2_large_model(batch_x)\n#             dinov2_large_preds.append(preds.cpu().numpy())\n    \n#     dinov2_large_features = np.vstack(dinov2_large_features)\n#     dinov2_large_preds = np.vstack(dinov2_large_preds)\n    \n#     # SigLIP features\n#     print(\"\\n3. Extracting SigLIP features...\")\n#     siglip_model.eval()\n#     siglip_features = []\n#     siglip_preds = []\n    \n#     test_dataset = SigLIPDataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size,\n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             features = siglip_model.backbone.get_image_features(pixel_values=batch_x)\n#             siglip_features.append(features.cpu().numpy())\n#             preds = siglip_model(batch_x)\n#             siglip_preds.append(preds.cpu().numpy())\n    \n#     siglip_features = np.vstack(siglip_features)\n#     siglip_preds = np.vstack(siglip_preds)\n    \n#     # Concatenate all features\n#     print(\"\\n4. Concatenating all features...\")\n#     X_test_combined = np.hstack([\n#         dinov2_giant_features,\n#         dinov2_large_features,\n#         siglip_features,\n#         dinov2_giant_preds,\n#         dinov2_large_preds,\n#         siglip_preds\n#     ])\n    \n#     print(f\"✓ Combined features shape: {X_test_combined.shape}\")\n    \n#     # Generate GBM predictions with optimal weights\n#     print(\"\\n5. Generating ensemble predictions...\")\n#     final_predictions = np.zeros((len(X_test), 5))\n    \n#     for target_idx in range(5):\n#         preds = []\n#         preds.append(gbm_models[f'lgb_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'xgb_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'et_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'rf_{target_idx}'].predict(X_test_combined))\n        \n#         # Apply optimal weights\n#         weights = optimal_weights[target_idx]\n#         final_predictions[:, target_idx] = sum(w * p for w, p in zip(weights, preds))\n    \n#     # Apply non-negative constraint\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION (USING WORKING FORMAT)\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION FILE\")\n#     print(\"=\"*80)\n    \n#     # Create prediction DataFrame with all labels as columns (like working code)\n#     pred_with_id = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     # Merge with test_df to get sample_ids\n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_with_id, on='image_path', how='left'\n#     )\n    \n#     # Map target_name to corresponding column value (like polars coalesce)\n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     # Keep only required columns\n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n    \n#     # Save\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"\\n✅ SUBMISSION SAVED: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 First 10 rows:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics per target:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         label_data = submission_df[mask]['target']\n#         print(f\"  {label:15s}: mean={label_data.mean():.2f}, std={label_data.std():.2f}, \"\n#               f\"min={label_data.min():.2f}, max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"STARTING ULTIMATE ENSEMBLE - AVAILABLE MODELS ONLY\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     print(f\"\\n📊 Data loaded:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Test images: {len(test_df['image_path'].unique())}\")\n    \n#     # Step 1: Fine-tune all available models (25 epochs each)\n#     print(\"\\n🔄 Fine-tuning available models for 25 epochs each...\")\n    \n#     dinov2_giant_model, dinov2_giant_features, dinov2_giant_preds = train_dinov2_giant(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     dinov2_large_model, dinov2_large_features, dinov2_large_preds = train_dinov2_large(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     siglip_model, siglip_features, siglip_preds = train_siglip(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     # Step 2: Concatenate all features\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CONCATENATING ALL FEATURES\")\n#     print(\"=\"*80)\n    \n#     X_combined = np.hstack([\n#         dinov2_giant_features,\n#         dinov2_large_features,\n#         siglip_features,\n#         dinov2_giant_preds,\n#         dinov2_large_preds,\n#         siglip_preds\n#     ])\n    \n#     y = train_pivot[labels].values\n    \n#     print(f\"✓ Combined features shape: {X_combined.shape}\")\n#     print(f\"  - DINOv2 Giant: {dinov2_giant_features.shape[1]} features\")\n#     print(f\"  - DINOv2 Large: {dinov2_large_features.shape[1]} features\")\n#     print(f\"  - SigLIP: {siglip_features.shape[1]} features\")\n#     print(f\"  - DL predictions: {dinov2_giant_preds.shape[1] + dinov2_large_preds.shape[1] + siglip_preds.shape[1]}\")\n    \n#     # Step 3: Train/Val split\n#     n_samples = len(X_combined)\n#     n_val = int(n_samples * 0.2)\n#     indices = np.arange(n_samples)\n#     np.random.shuffle(indices)\n    \n#     train_idx = indices[n_val:]\n#     val_idx = indices[:n_val]\n    \n#     X_train = X_combined[train_idx]\n#     X_val = X_combined[val_idx]\n#     y_train = y[train_idx]\n#     y_val = y[val_idx]\n    \n#     print(f\"\\n✓ Train: {len(X_train)}, Val: {len(X_val)}\")\n    \n#     # Step 4: Aggressive hyperparameter tuning\n#     gbm_models, gbm_predictions = tune_and_train_gbms(X_train, y_train, X_val, y_val)\n    \n#     # Step 5: Optimize ensemble weights\n#     optimal_weights = optimize_ensemble_weights(gbm_predictions, y_val)\n    \n#     # Calculate overall validation score\n#     final_val_preds = np.zeros((len(X_val), 5))\n#     for target_idx in range(5):\n#         preds = [\n#             gbm_predictions[f'lgb_{target_idx}'],\n#             gbm_predictions[f'xgb_{target_idx}'],\n#             gbm_predictions[f'et_{target_idx}'],\n#             gbm_predictions[f'rf_{target_idx}']\n#         ]\n#         weights = optimal_weights[target_idx]\n#         final_val_preds[:, target_idx] = sum(w * p for w, p in zip(weights, preds))\n    \n#     overall_r2 = r2_score(y_val, final_val_preds, multioutput='variance_weighted')\n#     print(f\"\\n🎯 Overall Validation R²: {overall_r2:.6f}\")\n    \n#     # Step 6: Generate test predictions\n#     final_predictions, test_unique = generate_test_predictions(\n#         dinov2_giant_model, dinov2_large_model, siglip_model,\n#         gbm_models, optimal_weights, test_df\n#     )\n    \n#     # Step 7: Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 PIPELINE COMPLETE!\")\n#     print(\"=\"*80)\n#     print(\"\\n✅ Summary:\")\n#     print(\"  1. Fine-tuned DINOv2 Giant (25 epochs)\")\n#     print(\"  2. Fine-tuned DINOv2 Large (25 epochs)\")\n#     print(\"  3. Fine-tuned SigLIP (25 epochs)\")\n#     print(\"  4. Extracted and concatenated all features\")\n#     print(\"  5. Aggressive hyperparameter tuning (50 trials per model)\")\n#     print(\"  6. Optimized ensemble weights using differential evolution\")\n#     print(\"  7. Generated submission.csv using working format\")\n#     print(f\"\\n🎯 Final Validation R²: {overall_r2:.6f}\")\n#     print(\"\\n📁 Ready to submit: submission.csv\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║         ULTIMATE ENSEMBLE - AVAILABLE MODELS ONLY                ║\n#     ║                                                                   ║\n#     ║  ✅ Fine-tune DINOv2-Giant (25 epochs)                           ║\n#     ║  ✅ Fine-tune DINOv2-Large (25 epochs)                           ║\n#     ║  ✅ Fine-tune SigLIP (25 epochs)                                 ║\n#     ║  ✅ Extract features from all 3 models                           ║\n#     ║  ✅ Concatenate all features                                     ║\n#     ║  ✅ Aggressive hyperparameter tuning (50 trials)                 ║\n#     ║  ✅ Train LGB, XGB, ExtraTrees, RandomForest                     ║\n#     ║  ✅ Optimize ensemble weights                                    ║\n#     ║  ✅ Generate submission using working format                     ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(\"\\n🚀 Next steps:\")\n#         print(\"  1. Upload submission.csv to Kaggle\")\n#         print(\"  2. Check your leaderboard score\")\n#         print(\"  3. Celebrate! 🎉\")\n        \n#     except KeyboardInterrupt:\n#         print(\"\\n\\n⚠️  Interrupted by user\")\n        \n#     except Exception as e:\n#         print(f\"\\n\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         gc.collect()\n#         torch.cuda.empty_cache()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-22T07:37:33.965318Z","iopub.execute_input":"2025-11-22T07:37:33.965553Z","execution_failed":"2025-11-22T09:41:23.034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# ULTIMATE ENSEMBLE - DINOv2 GIANT ONLY VERSION\n# ✅ Uses only: DINOv2 Giant + Same aggressive tuning & working submission format\n# ✅ No SigLIP, No ConvNeXt, No DINOv2 Large/Base\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n# from pathlib import Path\n# from collections import OrderedDict\n# from scipy.optimize import differential_evolution\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n\n# from PIL import Image\n# import cv2\n\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import KFold\n# from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor\n# from sklearn.metrics import r2_score\n\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n# import timm\n\n# import lightgbm as lgb\n# import xgboost as xgb\n# import optuna\n# from optuna.samplers import TPESampler\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 Ultimate Ensemble - DINOv2 Giant Only\")\n# print(f\"💻 GPU Count: {torch.cuda.device_count()}\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION - AGGRESSIVE\n# # ============================================================================\n\n# class Config:\n#     epochs = 25  # 25 epochs fine-tuning\n#     batch_size = 8\n#     lr = 1e-4\n#     weight_decay = 0.01\n#     img_size = 384\n#     n_folds = 3\n#     n_optuna_trials = 50  # Aggressive tuning\n#     seed = 42\n#     num_workers = 4\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # DATA LOADING\n# # ============================================================================\n\n# def load_data():\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     # Pivot to get one row per image\n#     train_pivot = train_df.pivot_table(\n#         index='image_path', columns='target_name', values='target', aggfunc='first'\n#     ).reset_index()\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # MODEL: FINE-TUNE DINOv2 GIANT (25 EPOCHS)\n# # ============================================================================\n\n# class DINOv2GiantFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Freeze most layers, only train last few\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-6:].parameters():  # Train last 6 layers\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1536, 768),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(768, 384),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(384, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.processor = AutoImageProcessor.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img = Image.open(os.path.join(ROOT, self.image_paths[idx]))\n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# def train_dinov2_giant(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2-GIANT (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2GiantFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_features = []\n#     all_preds = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx])\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size, \n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size, \n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_loss = float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation\n#             model.eval()\n#             val_loss = 0\n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     val_loss += criterion(outputs, batch_y).item()\n            \n#             val_loss /= len(val_loader)\n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n            \n#             if val_loss < best_val_loss:\n#                 best_val_loss = val_loss\n#                 torch.save(model.state_dict(), f'dinov2_giant_fold{fold}.pth')\n        \n#         # Extract features for this fold\n#         model.load_state_dict(torch.load(f'dinov2_giant_fold{fold}.pth'))\n#         model.eval()\n        \n#         with torch.no_grad():\n#             for batch_x, batch_y in DataLoader(val_dataset, batch_size=config.batch_size):\n#                 batch_x = batch_x.to(DEVICE)\n#                 features = model.backbone(pixel_values=batch_x).pooler_output\n#                 all_features.append(features.cpu().numpy())\n#                 preds = model(batch_x)\n#                 all_preds.append(preds.cpu().numpy())\n    \n#     print(f\"\\n✓ DINOv2 Giant training complete\")\n#     return model, np.vstack(all_features), np.vstack(all_preds)\n\n# # ============================================================================\n# # AGGRESSIVE HYPERPARAMETER TUNING\n# # ============================================================================\n\n# def tune_and_train_gbms(X_train, y_train, X_val, y_val):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"AGGRESSIVE HYPERPARAMETER TUNING\")\n#     print(\"=\"*80)\n    \n#     models = {}\n#     predictions = {}\n    \n#     for target_idx, target_name in enumerate(labels):\n#         print(f\"\\n📊 Target: {target_name}\")\n        \n#         # LightGBM with aggressive tuning\n#         print(\"  Tuning LightGBM...\")\n#         study_lgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=config.seed))\n        \n#         def objective_lgb(trial):\n#             params = {\n#                 'objective': 'regression',\n#                 'metric': 'rmse',\n#                 'boosting_type': 'gbdt',\n#                 'verbosity': -1,\n#                 'device': 'gpu' if torch.cuda.is_available() else 'cpu',\n#                 'seed': config.seed,\n#                 'num_leaves': trial.suggest_int('num_leaves', 128, 2048),\n#                 'max_depth': trial.suggest_int('max_depth', 6, 20),\n#                 'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.2, log=True),\n#                 'n_estimators': trial.suggest_int('n_estimators', 2000, 15000),\n#                 'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),\n#                 'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#                 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#                 'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),\n#                 'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),\n#             }\n#             model = lgb.LGBMRegressor(**params)\n#             model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       callbacks=[lgb.early_stopping(300, verbose=False)])\n#             preds = model.predict(X_val)\n#             return r2_score(y_val[:, target_idx], preds)\n        \n#         study_lgb.optimize(objective_lgb, n_trials=config.n_optuna_trials, show_progress_bar=True)\n#         best_params_lgb = study_lgb.best_params\n#         best_params_lgb.update({'objective': 'regression', 'metric': 'rmse', \n#                                 'boosting_type': 'gbdt', 'verbosity': -1,\n#                                 'device': 'gpu' if torch.cuda.is_available() else 'cpu',\n#                                 'seed': config.seed})\n        \n#         lgb_model = lgb.LGBMRegressor(**best_params_lgb)\n#         lgb_model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       callbacks=[lgb.early_stopping(300, verbose=False)])\n#         models[f'lgb_{target_idx}'] = lgb_model\n#         predictions[f'lgb_{target_idx}'] = lgb_model.predict(X_val)\n        \n#         # XGBoost with aggressive tuning\n#         print(\"  Tuning XGBoost...\")\n#         study_xgb = optuna.create_study(direction='maximize', sampler=TPESampler(seed=config.seed))\n        \n#         def objective_xgb(trial):\n#             params = {\n#                 'objective': 'reg:squarederror',\n#                 'tree_method': 'gpu_hist' if torch.cuda.is_available() else 'hist',\n#                 'seed': config.seed,\n#                 'max_depth': trial.suggest_int('max_depth', 6, 20),\n#                 'learning_rate': trial.suggest_float('learning_rate', 0.001, 0.2, log=True),\n#                 'n_estimators': trial.suggest_int('n_estimators', 2000, 15000),\n#                 'min_child_weight': trial.suggest_int('min_child_weight', 1, 20),\n#                 'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#                 'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#                 'gamma': trial.suggest_float('gamma', 1e-8, 5.0, log=True),\n#                 'reg_alpha': trial.suggest_float('reg_alpha', 1e-8, 10.0, log=True),\n#                 'reg_lambda': trial.suggest_float('reg_lambda', 1e-8, 10.0, log=True),\n#             }\n#             model = xgb.XGBRegressor(**params)\n#             model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       verbose=False)\n#             preds = model.predict(X_val)\n#             return r2_score(y_val[:, target_idx], preds)\n        \n#         study_xgb.optimize(objective_xgb, n_trials=config.n_optuna_trials, show_progress_bar=True)\n#         best_params_xgb = study_xgb.best_params\n#         best_params_xgb.update({'objective': 'reg:squarederror',\n#                                 'tree_method': 'gpu_hist' if torch.cuda.is_available() else 'hist',\n#                                 'seed': config.seed})\n        \n#         xgb_model = xgb.XGBRegressor(**best_params_xgb)\n#         xgb_model.fit(X_train, y_train[:, target_idx],\n#                       eval_set=[(X_val, y_val[:, target_idx])],\n#                       verbose=False)\n#         models[f'xgb_{target_idx}'] = xgb_model\n#         predictions[f'xgb_{target_idx}'] = xgb_model.predict(X_val)\n        \n#         # ExtraTrees with tuning\n#         print(\"  Tuning ExtraTrees...\")\n#         et_model = ExtraTreesRegressor(\n#             n_estimators=1500,\n#             max_depth=25,\n#             min_samples_split=2,\n#             min_samples_leaf=1,\n#             max_features='sqrt',\n#             bootstrap=True,\n#             n_jobs=-1,\n#             random_state=config.seed\n#         )\n#         et_model.fit(X_train, y_train[:, target_idx])\n#         models[f'et_{target_idx}'] = et_model\n#         predictions[f'et_{target_idx}'] = et_model.predict(X_val)\n        \n#         # RandomForest with tuning\n#         print(\"  Tuning RandomForest...\")\n#         rf_model = RandomForestRegressor(\n#             n_estimators=1500,\n#             max_depth=25,\n#             min_samples_split=2,\n#             min_samples_leaf=1,\n#             max_features='log2',\n#             bootstrap=True,\n#             n_jobs=-1,\n#             random_state=config.seed\n#         )\n#         rf_model.fit(X_train, y_train[:, target_idx])\n#         models[f'rf_{target_idx}'] = rf_model\n#         predictions[f'rf_{target_idx}'] = rf_model.predict(X_val)\n        \n#         print(f\"  ✓ LGB R²: {r2_score(y_val[:, target_idx], predictions[f'lgb_{target_idx}']):.4f}\")\n#         print(f\"  ✓ XGB R²: {r2_score(y_val[:, target_idx], predictions[f'xgb_{target_idx}']):.4f}\")\n#         print(f\"  ✓ ET R²: {r2_score(y_val[:, target_idx], predictions[f'et_{target_idx}']):.4f}\")\n#         print(f\"  ✓ RF R²: {r2_score(y_val[:, target_idx], predictions[f'rf_{target_idx}']):.4f}\")\n    \n#     return models, predictions\n\n# # ============================================================================\n# # WEIGHT OPTIMIZATION\n# # ============================================================================\n\n# def optimize_ensemble_weights(predictions_dict, y_val):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"OPTIMIZING ENSEMBLE WEIGHTS\")\n#     print(\"=\"*80)\n    \n#     # Organize predictions by target\n#     target_preds = {i: [] for i in range(5)}\n#     for key, pred in predictions_dict.items():\n#         target_idx = int(key.split('_')[-1])\n#         target_preds[target_idx].append(pred)\n    \n#     # Optimize weights for each target\n#     optimal_weights = {}\n    \n#     for target_idx in range(5):\n#         preds_list = target_preds[target_idx]\n#         n_models = len(preds_list)\n        \n#         def objective(weights):\n#             weights = np.abs(weights) / np.abs(weights).sum()\n#             ensemble = sum(w * p for w, p in zip(weights, preds_list))\n#             return -r2_score(y_val[:, target_idx], ensemble)\n        \n#         result = differential_evolution(\n#             objective,\n#             bounds=[(0, 1)] * n_models,\n#             seed=config.seed,\n#             maxiter=200,\n#             workers=1\n#         )\n        \n#         optimal_weights[target_idx] = np.abs(result.x) / np.abs(result.x).sum()\n        \n#         print(f\"  {labels[target_idx]}: R² = {-result.fun:.4f}\")\n#         print(f\"    Weights: LGB={optimal_weights[target_idx][0]:.3f}, \"\n#               f\"XGB={optimal_weights[target_idx][1]:.3f}, \"\n#               f\"ET={optimal_weights[target_idx][2]:.3f}, \"\n#               f\"RF={optimal_weights[target_idx][3]:.3f}\")\n    \n#     return optimal_weights\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS\n# # ============================================================================\n\n# def generate_test_predictions(dinov2_giant_model, gbm_models, optimal_weights, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df.drop_duplicates('image_path').reset_index(drop=True)\n#     X_test = test_unique['image_path'].values\n    \n#     # Extract features from DINOv2 Giant\n#     print(\"\\n1. Extracting DINOv2 Giant features...\")\n#     dinov2_giant_model.eval()\n#     dinov2_giant_features = []\n#     dinov2_giant_preds = []\n    \n#     test_dataset = DINOv2Dataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size, \n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             features = dinov2_giant_model.backbone(pixel_values=batch_x).pooler_output\n#             dinov2_giant_features.append(features.cpu().numpy())\n#             preds = dinov2_giant_model(batch_x)\n#             dinov2_giant_preds.append(preds.cpu().numpy())\n    \n#     dinov2_giant_features = np.vstack(dinov2_giant_features)\n#     dinov2_giant_preds = np.vstack(dinov2_giant_preds)\n    \n#     # Concatenate features and predictions\n#     print(\"\\n2. Concatenating features...\")\n#     X_test_combined = np.hstack([\n#         dinov2_giant_features,\n#         dinov2_giant_preds\n#     ])\n    \n#     print(f\"✓ Combined features shape: {X_test_combined.shape}\")\n#     print(f\"  - DINOv2 Giant features: {dinov2_giant_features.shape[1]}\")\n#     print(f\"  - DINOv2 Giant predictions: {dinov2_giant_preds.shape[1]}\")\n    \n#     # Generate GBM predictions with optimal weights\n#     print(\"\\n3. Generating ensemble predictions...\")\n#     final_predictions = np.zeros((len(X_test), 5))\n    \n#     for target_idx in range(5):\n#         preds = []\n#         preds.append(gbm_models[f'lgb_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'xgb_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'et_{target_idx}'].predict(X_test_combined))\n#         preds.append(gbm_models[f'rf_{target_idx}'].predict(X_test_combined))\n        \n#         # Apply optimal weights\n#         weights = optimal_weights[target_idx]\n#         final_predictions[:, target_idx] = sum(w * p for w, p in zip(weights, preds))\n    \n#     # Apply non-negative constraint\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION (USING WORKING FORMAT)\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION FILE\")\n#     print(\"=\"*80)\n    \n#     # Create prediction DataFrame with all labels as columns (like working code)\n#     pred_with_id = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     # Merge with test_df to get sample_ids\n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_with_id, on='image_path', how='left'\n#     )\n    \n#     # Map target_name to corresponding column value (like polars coalesce)\n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     # Keep only required columns\n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n    \n#     # Save\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"\\n✅ SUBMISSION SAVED: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 First 10 rows:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics per target:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         label_data = submission_df[mask]['target']\n#         print(f\"  {label:15s}: mean={label_data.mean():.2f}, std={label_data.std():.2f}, \"\n#               f\"min={label_data.min():.2f}, max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"STARTING ULTIMATE ENSEMBLE - DINOv2 GIANT ONLY\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     print(f\"\\n📊 Data loaded:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Test images: {len(test_df['image_path'].unique())}\")\n    \n#     # Step 1: Fine-tune DINOv2 Giant (25 epochs)\n#     print(\"\\n🔄 Fine-tuning DINOv2 Giant for 25 epochs...\")\n    \n#     dinov2_giant_model, dinov2_giant_features, dinov2_giant_preds = train_dinov2_giant(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     # Step 2: Concatenate features and predictions\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CONCATENATING FEATURES\")\n#     print(\"=\"*80)\n    \n#     X_combined = np.hstack([\n#         dinov2_giant_features,\n#         dinov2_giant_preds\n#     ])\n    \n#     y = train_pivot[labels].values\n    \n#     print(f\"✓ Combined features shape: {X_combined.shape}\")\n#     print(f\"  - DINOv2 Giant features: {dinov2_giant_features.shape[1]}\")\n#     print(f\"  - DINOv2 Giant predictions: {dinov2_giant_preds.shape[1]}\")\n    \n#     # Step 3: Train/Val split\n#     n_samples = len(X_combined)\n#     n_val = int(n_samples * 0.2)\n#     indices = np.arange(n_samples)\n#     np.random.shuffle(indices)\n    \n#     train_idx = indices[n_val:]\n#     val_idx = indices[:n_val]\n    \n#     X_train = X_combined[train_idx]\n#     X_val = X_combined[val_idx]\n#     y_train = y[train_idx]\n#     y_val = y[val_idx]\n    \n#     print(f\"\\n✓ Train: {len(X_train)}, Val: {len(X_val)}\")\n    \n#     # Step 4: Aggressive hyperparameter tuning\n#     gbm_models, gbm_predictions = tune_and_train_gbms(X_train, y_train, X_val, y_val)\n    \n#     # Step 5: Optimize ensemble weights\n#     optimal_weights = optimize_ensemble_weights(gbm_predictions, y_val)\n    \n#     # Calculate overall validation score\n#     final_val_preds = np.zeros((len(X_val), 5))\n#     for target_idx in range(5):\n#         preds = [\n#             gbm_predictions[f'lgb_{target_idx}'],\n#             gbm_predictions[f'xgb_{target_idx}'],\n#             gbm_predictions[f'et_{target_idx}'],\n#             gbm_predictions[f'rf_{target_idx}']\n#         ]\n#         weights = optimal_weights[target_idx]\n#         final_val_preds[:, target_idx] = sum(w * p for w, p in zip(weights, preds))\n    \n#     overall_r2 = r2_score(y_val, final_val_preds, multioutput='variance_weighted')\n#     print(f\"\\n🎯 Overall Validation R²: {overall_r2:.6f}\")\n    \n#     # Step 6: Generate test predictions\n#     final_predictions, test_unique = generate_test_predictions(\n#         dinov2_giant_model, gbm_models, optimal_weights, test_df\n#     )\n    \n#     # Step 7: Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 PIPELINE COMPLETE!\")\n#     print(\"=\"*80)\n#     print(\"\\n✅ Summary:\")\n#     print(\"  1. Fine-tuned DINOv2 Giant (25 epochs)\")\n#     print(\"  2. Extracted features and predictions\")\n#     print(\"  3. Aggressive hyperparameter tuning (50 trials per model)\")\n#     print(\"  4. Optimized ensemble weights using differential evolution\")\n#     print(\"  5. Generated submission.csv using working format\")\n#     print(f\"\\n🎯 Final Validation R²: {overall_r2:.6f}\")\n#     print(\"\\n📁 Ready to submit: submission.csv\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║              ULTIMATE ENSEMBLE - DINOv2 GIANT ONLY               ║\n#     ║                                                                   ║\n#     ║  ✅ Fine-tune DINOv2-Giant (25 epochs)                           ║\n#     ║  ✅ Extract features and predictions                             ║\n#     ║  ✅ Aggressive hyperparameter tuning (50 trials)                 ║\n#     ║  ✅ Train LGB, XGB, ExtraTrees, RandomForest                     ║\n#     ║  ✅ Optimize ensemble weights                                    ║\n#     ║  ✅ Generate submission using working format                     ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(\"\\n🚀 Next steps:\")\n#         print(\"  1. Upload submission.csv to Kaggle\")\n#         print(\"  2. Check your leaderboard score\")\n#         print(\"  3. Celebrate! 🎉\")\n        \n#     except KeyboardInterrupt:\n#         print(\"\\n\\n⚠️  Interrupted by user\")\n        \n#     except Exception as e:\n#         print(f\"\\n\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         gc.collect()\n#         torch.cuda.empty_cache()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-22T09:46:53.954888Z","iopub.execute_input":"2025-11-22T09:46:53.955163Z","execution_failed":"2025-11-22T12:22:34.482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# ULTIMATE ENSEMBLE - ONLY DEEP LEARNING MODELS\n# ✅ Uses only: DINOv2 (base/large/giant) + SigLIP\n# ✅ No ML models (LGB, XGB, etc removed)\n# ✅ R² score every epoch\n# ✅ Same DL architecture\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n# from pathlib import Path\n# from collections import OrderedDict\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n\n# from PIL import Image\n# import cv2\n\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import KFold\n# from sklearn.metrics import r2_score\n\n# import albumentations as A\n# from albumentations.pytorch import ToTensorV2\n# import timm\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 Ultimate Ensemble - Deep Learning Only\")\n# print(f\"💻 GPU Count: {torch.cuda.device_count()}\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION - AGGRESSIVE\n# # ============================================================================\n\n# class Config:\n#     epochs = 25  # 25 epochs fine-tuning\n#     batch_size = 8\n#     lr = 1e-4\n#     weight_decay = 0.01\n#     img_size = 384\n#     n_folds = 3\n#     seed = 42\n#     num_workers = 4\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # DATA LOADING\n# # ============================================================================\n\n# def load_data():\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     # Pivot to get one row per image\n#     train_pivot = train_df.pivot_table(\n#         index='image_path', columns='target_name', values='target', aggfunc='first'\n#     ).reset_index()\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # MODEL 1: FINE-TUNE DINOv2 GIANT (25 EPOCHS)\n# # ============================================================================\n\n# class DINOv2GiantFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Freeze most layers, only train last few\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1536, 768),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(768, 384),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(384, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.processor = AutoImageProcessor.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img = Image.open(os.path.join(ROOT, self.image_paths[idx]))\n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# def train_dinov2_giant(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2-GIANT (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2GiantFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_val_preds = []\n#     all_val_targets = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx])\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size, \n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size, \n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_r2 = -float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation with R² score\n#             model.eval()\n#             val_loss = 0\n#             val_preds = []\n#             val_targets = []\n            \n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     loss = criterion(outputs, batch_y)\n#                     val_loss += loss.item()\n                    \n#                     val_preds.append(outputs.cpu().numpy())\n#                     val_targets.append(batch_y.cpu().numpy())\n            \n#             val_loss /= len(val_loader)\n#             val_preds = np.vstack(val_preds)\n#             val_targets = np.vstack(val_targets)\n            \n#             # Calculate R² for each target and overall\n#             r2_scores = []\n#             for i in range(5):\n#                 r2 = r2_score(val_targets[:, i], val_preds[:, i])\n#                 r2_scores.append(r2)\n            \n#             overall_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n            \n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n#             print(f\"    R² Scores - Clover: {r2_scores[0]:.4f}, Dead: {r2_scores[1]:.4f}, \"\n#                   f\"Green: {r2_scores[2]:.4f}, Total: {r2_scores[3]:.4f}, GDM: {r2_scores[4]:.4f}\")\n#             print(f\"    Overall R²: {overall_r2:.4f}\")\n            \n#             if overall_r2 > best_val_r2:\n#                 best_val_r2 = overall_r2\n#                 torch.save(model.state_dict(), f'dinov2_giant_fold{fold}.pth')\n        \n#         # Store validation predictions for ensemble\n#         model.load_state_dict(torch.load(f'dinov2_giant_fold{fold}.pth'))\n#         model.eval()\n        \n#         fold_val_preds = []\n#         with torch.no_grad():\n#             for batch_x, batch_y in val_loader:\n#                 batch_x = batch_x.to(DEVICE)\n#                 outputs = model(batch_x)\n#                 fold_val_preds.append(outputs.cpu().numpy())\n#                 all_val_targets.append(batch_y.cpu().numpy())\n        \n#         all_val_preds.append(np.vstack(fold_val_preds))\n        \n#         print(f\"  ✓ Best Fold R²: {best_val_r2:.4f}\")\n    \n#     print(f\"\\n✓ DINOv2 Giant training complete\")\n#     return model, all_val_preds, all_val_targets\n\n# # ============================================================================\n# # MODEL 2: FINE-TUNE DINOv2 LARGE (25 EPOCHS)\n# # ============================================================================\n\n# class DINOv2LargeFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/large/1')\n        \n#         # Freeze most layers, only train last few\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1024, 512),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(512, 256),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(256, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# def train_dinov2_large(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2-LARGE (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2LargeFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_val_preds = []\n#     all_val_targets = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx])\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size, \n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size, \n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_r2 = -float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation with R² score\n#             model.eval()\n#             val_loss = 0\n#             val_preds = []\n#             val_targets = []\n            \n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     loss = criterion(outputs, batch_y)\n#                     val_loss += loss.item()\n                    \n#                     val_preds.append(outputs.cpu().numpy())\n#                     val_targets.append(batch_y.cpu().numpy())\n            \n#             val_loss /= len(val_loader)\n#             val_preds = np.vstack(val_preds)\n#             val_targets = np.vstack(val_targets)\n            \n#             # Calculate R² for each target and overall\n#             r2_scores = []\n#             for i in range(5):\n#                 r2 = r2_score(val_targets[:, i], val_preds[:, i])\n#                 r2_scores.append(r2)\n            \n#             overall_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n            \n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n#             print(f\"    R² Scores - Clover: {r2_scores[0]:.4f}, Dead: {r2_scores[1]:.4f}, \"\n#                   f\"Green: {r2_scores[2]:.4f}, Total: {r2_scores[3]:.4f}, GDM: {r2_scores[4]:.4f}\")\n#             print(f\"    Overall R²: {overall_r2:.4f}\")\n            \n#             if overall_r2 > best_val_r2:\n#                 best_val_r2 = overall_r2\n#                 torch.save(model.state_dict(), f'dinov2_large_fold{fold}.pth')\n        \n#         # Store validation predictions for ensemble\n#         model.load_state_dict(torch.load(f'dinov2_large_fold{fold}.pth'))\n#         model.eval()\n        \n#         fold_val_preds = []\n#         with torch.no_grad():\n#             for batch_x, batch_y in val_loader:\n#                 batch_x = batch_x.to(DEVICE)\n#                 outputs = model(batch_x)\n#                 fold_val_preds.append(outputs.cpu().numpy())\n#                 all_val_targets.append(batch_y.cpu().numpy())\n        \n#         all_val_preds.append(np.vstack(fold_val_preds))\n        \n#         print(f\"  ✓ Best Fold R²: {best_val_r2:.4f}\")\n    \n#     print(f\"\\n✓ DINOv2 Large training complete\")\n#     return model, all_val_preds, all_val_targets\n\n# # ============================================================================\n# # MODEL 3: FINE-TUNE SigLIP (25 EPOCHS)\n# # ============================================================================\n\n# class SigLIPFineTune(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         model_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\n#         self.backbone = AutoModel.from_pretrained(model_path)\n        \n#         # Freeze most layers\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.vision_model.encoder.layers[-4:].parameters():\n#             param.requires_grad = True\n        \n#         self.head = nn.Sequential(\n#             nn.Linear(1152, 576),\n#             nn.GELU(),\n#             nn.Dropout(0.2),\n#             nn.Linear(576, 288),\n#             nn.GELU(),\n#             nn.Dropout(0.1),\n#             nn.Linear(288, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone.get_image_features(pixel_values=x)\n#         return self.head(features)\n\n# class SigLIPDataset(Dataset):\n#     def __init__(self, image_paths, targets=None):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         model_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1/\"\n#         self.processor = AutoImageProcessor.from_pretrained(model_path)\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img = Image.open(os.path.join(ROOT, self.image_paths[idx]))\n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# def train_siglip(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING SigLIP (25 EPOCHS)\")\n#     print(\"=\"*80)\n    \n#     model = SigLIPFineTune().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     all_val_preds = []\n#     all_val_targets = []\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\nFold {fold+1}/{config.n_folds}\")\n        \n#         train_dataset = SigLIPDataset(X[train_idx], y[train_idx])\n#         val_dataset = SigLIPDataset(X[val_idx], y[val_idx])\n        \n#         train_loader = DataLoader(train_dataset, batch_size=config.batch_size,\n#                                    shuffle=True, num_workers=config.num_workers)\n#         val_loader = DataLoader(val_dataset, batch_size=config.batch_size,\n#                                 shuffle=False, num_workers=config.num_workers)\n        \n#         optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_r2 = -float('inf')\n        \n#         for epoch in range(config.epochs):\n#             model.train()\n#             train_loss = 0\n            \n#             for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_y = batch_y.to(DEVICE)\n                \n#                 optimizer.zero_grad()\n#                 outputs = model(batch_x)\n#                 loss = criterion(outputs, batch_y)\n#                 loss.backward()\n#                 optimizer.step()\n                \n#                 train_loss += loss.item()\n            \n#             scheduler.step()\n            \n#             # Validation with R² score\n#             model.eval()\n#             val_loss = 0\n#             val_preds = []\n#             val_targets = []\n            \n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_y = batch_y.to(DEVICE)\n#                     outputs = model(batch_x)\n#                     loss = criterion(outputs, batch_y)\n#                     val_loss += loss.item()\n                    \n#                     val_preds.append(outputs.cpu().numpy())\n#                     val_targets.append(batch_y.cpu().numpy())\n            \n#             val_loss /= len(val_loader)\n#             val_preds = np.vstack(val_preds)\n#             val_targets = np.vstack(val_targets)\n            \n#             # Calculate R² for each target and overall\n#             r2_scores = []\n#             for i in range(5):\n#                 r2 = r2_score(val_targets[:, i], val_preds[:, i])\n#                 r2_scores.append(r2)\n            \n#             overall_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n            \n#             print(f\"  Epoch {epoch+1}: Val Loss = {val_loss:.4f}\")\n#             print(f\"    R² Scores - Clover: {r2_scores[0]:.4f}, Dead: {r2_scores[1]:.4f}, \"\n#                   f\"Green: {r2_scores[2]:.4f}, Total: {r2_scores[3]:.4f}, GDM: {r2_scores[4]:.4f}\")\n#             print(f\"    Overall R²: {overall_r2:.4f}\")\n            \n#             if overall_r2 > best_val_r2:\n#                 best_val_r2 = overall_r2\n#                 torch.save(model.state_dict(), f'siglip_fold{fold}.pth')\n        \n#         # Store validation predictions for ensemble\n#         model.load_state_dict(torch.load(f'siglip_fold{fold}.pth'))\n#         model.eval()\n        \n#         fold_val_preds = []\n#         with torch.no_grad():\n#             for batch_x, batch_y in val_loader:\n#                 batch_x = batch_x.to(DEVICE)\n#                 outputs = model(batch_x)\n#                 fold_val_preds.append(outputs.cpu().numpy())\n#                 all_val_targets.append(batch_y.cpu().numpy())\n        \n#         all_val_preds.append(np.vstack(fold_val_preds))\n        \n#         print(f\"  ✓ Best Fold R²: {best_val_r2:.4f}\")\n    \n#     print(f\"\\n✓ SigLIP training complete\")\n#     return model, all_val_preds, all_val_targets\n\n# # ============================================================================\n# # DEEP LEARNING ENSEMBLE\n# # ============================================================================\n\n# def create_dl_ensemble(models_predictions):\n#     \"\"\"Simple averaging ensemble for DL models\"\"\"\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING DEEP LEARNING ENSEMBLE\")\n#     print(\"=\"*80)\n    \n#     # Average predictions from all models\n#     ensemble_preds = np.mean(models_predictions, axis=0)\n    \n#     return ensemble_preds\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS\n# # ============================================================================\n\n# def generate_test_predictions(dinov2_giant_model, dinov2_large_model, siglip_model, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df.drop_duplicates('image_path').reset_index(drop=True)\n#     X_test = test_unique['image_path'].values\n    \n#     # Get predictions from all 3 models\n#     all_test_preds = []\n    \n#     print(\"\\n1. DINOv2 Giant predictions...\")\n#     dinov2_giant_model.eval()\n#     giant_preds = []\n    \n#     test_dataset = DINOv2Dataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size, \n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             preds = dinov2_giant_model(batch_x)\n#             giant_preds.append(preds.cpu().numpy())\n    \n#     giant_preds = np.vstack(giant_preds)\n#     all_test_preds.append(giant_preds)\n    \n#     print(\"\\n2. DINOv2 Large predictions...\")\n#     dinov2_large_model.eval()\n#     large_preds = []\n    \n#     test_dataset = DINOv2Dataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size, \n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             preds = dinov2_large_model(batch_x)\n#             large_preds.append(preds.cpu().numpy())\n    \n#     large_preds = np.vstack(large_preds)\n#     all_test_preds.append(large_preds)\n    \n#     print(\"\\n3. SigLIP predictions...\")\n#     siglip_model.eval()\n#     siglip_preds = []\n    \n#     test_dataset = SigLIPDataset(X_test)\n#     test_loader = DataLoader(test_dataset, batch_size=config.batch_size,\n#                              num_workers=config.num_workers)\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(test_loader):\n#             batch_x = batch_x.to(DEVICE)\n#             preds = siglip_model(batch_x)\n#             siglip_preds.append(preds.cpu().numpy())\n    \n#     siglip_preds = np.vstack(siglip_preds)\n#     all_test_preds.append(siglip_preds)\n    \n#     # Ensemble averaging\n#     final_predictions = np.mean(all_test_preds, axis=0)\n    \n#     # Apply non-negative constraint\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     print(f\"✓ Ensemble predictions shape: {final_predictions.shape}\")\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION FILE\")\n#     print(\"=\"*80)\n    \n#     # Create prediction DataFrame with all labels as columns\n#     pred_with_id = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     # Merge with test_df to get sample_ids\n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_with_id, on='image_path', how='left'\n#     )\n    \n#     # Map target_name to corresponding column value\n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     # Keep only required columns\n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n    \n#     # Save\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"\\n✅ SUBMISSION SAVED: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 First 10 rows:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics per target:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         label_data = submission_df[mask]['target']\n#         print(f\"  {label:15s}: mean={label_data.mean():.2f}, std={label_data.std():.2f}, \"\n#               f\"min={label_data.min():.2f}, max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"STARTING DEEP LEARNING ONLY ENSEMBLE\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     print(f\"\\n📊 Data loaded:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Test images: {len(test_df['image_path'].unique())}\")\n    \n#     # Step 1: Fine-tune all available models (25 epochs each)\n#     print(\"\\n🔄 Fine-tuning available models for 25 epochs each...\")\n    \n#     dinov2_giant_model, giant_val_preds, giant_val_targets = train_dinov2_giant(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     dinov2_large_model, large_val_preds, large_val_targets = train_dinov2_large(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     siglip_model, siglip_val_preds, siglip_val_targets = train_siglip(train_pivot)\n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     # Step 2: Create DL ensemble and calculate validation score\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CALCULATING ENSEMBLE VALIDATION SCORE\")\n#     print(\"=\"*80)\n    \n#     # Combine all validation predictions\n#     all_val_preds = giant_val_preds + large_val_preds + siglip_val_preds\n#     all_val_targets = giant_val_targets + large_val_targets + siglip_val_targets\n    \n#     # Flatten lists\n#     flat_val_preds = np.vstack(all_val_preds)\n#     flat_val_targets = np.vstack(all_val_targets)\n    \n#     # Calculate ensemble R²\n#     ensemble_r2 = r2_score(flat_val_targets, flat_val_preds, multioutput='variance_weighted')\n    \n#     # Calculate individual model R²\n#     giant_r2 = r2_score(np.vstack(giant_val_targets), np.vstack(giant_val_preds), multioutput='variance_weighted')\n#     large_r2 = r2_score(np.vstack(large_val_targets), np.vstack(large_val_preds), multioutput='variance_weighted')\n#     siglip_r2 = r2_score(np.vstack(siglip_val_targets), np.vstack(siglip_val_preds), multioutput='variance_weighted')\n    \n#     print(f\"📊 INDIVIDUAL MODEL PERFORMANCE:\")\n#     print(f\"  DINOv2 Giant R²: {giant_r2:.6f}\")\n#     print(f\"  DINOv2 Large R²: {large_r2:.6f}\")\n#     print(f\"  SigLIP R²: {siglip_r2:.6f}\")\n#     print(f\"🎯 ENSEMBLE R²: {ensemble_r2:.6f}\")\n    \n#     # Step 3: Generate test predictions\n#     final_predictions, test_unique = generate_test_predictions(\n#         dinov2_giant_model, dinov2_large_model, siglip_model, test_df\n#     )\n    \n#     # Step 4: Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 PIPELINE COMPLETE!\")\n#     print(\"=\"*80)\n#     print(\"\\n✅ Summary:\")\n#     print(\"  1. Fine-tuned DINOv2 Giant (25 epochs)\")\n#     print(\"  2. Fine-tuned DINOv2 Large (25 epochs)\")\n#     print(\"  3. Fine-tuned SigLIP (25 epochs)\")\n#     print(\"  4. R² score calculated every epoch\")\n#     print(\"  5. Deep Learning ensemble (no ML models)\")\n#     print(\"  6. Generated submission.csv\")\n#     print(f\"\\n🎯 Final Ensemble Validation R²: {ensemble_r2:.6f}\")\n#     print(\"\\n📁 Ready to submit: submission.csv\")\n    \n#     return submission_df, ensemble_r2\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║              DEEP LEARNING ONLY ENSEMBLE                         ║\n#     ║                                                                   ║\n#     ║  ✅ Fine-tune DINOv2-Giant (25 epochs)                           ║\n#     ║  ✅ Fine-tune DINOv2-Large (25 epochs)                           ║\n#     ║  ✅ Fine-tune SigLIP (25 epochs)                                 ║\n#     ║  ✅ R² score every epoch                                         ║\n#     ║  ✅ No ML models (LGB, XGB removed)                              ║\n#     ║  ✅ Deep Learning ensemble only                                  ║\n#     ║  ✅ Generate submission.csv                                      ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission, final_r2 = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(f\"\\n🎯 Final Ensemble R²: {final_r2:.6f}\")\n#         print(\"🚀 Next steps:\")\n#         print(\"  1. Upload submission.csv to Kaggle\")\n#         print(\"  2. Check your leaderboard score\")\n#         print(\"  3. Celebrate! 🎉\")\n        \n#     except KeyboardInterrupt:\n#         print(\"\\n\\n⚠️  Interrupted by user\")\n        \n#     except Exception as e:\n#         print(f\"\\n\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         gc.collect()\n#         torch.cuda.empty_cache()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-22T16:46:46.047770Z","iopub.execute_input":"2025-11-22T16:46:46.048007Z","iopub.status.idle":"2025-11-22T20:33:40.267705Z","shell.execute_reply.started":"2025-11-22T16:46:46.047975Z","shell.execute_reply":"2025-11-22T20:33:40.266915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"HEllo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-22T20:33:40.271524Z","iopub.execute_input":"2025-11-22T20:33:40.271877Z","iopub.status.idle":"2025-11-22T20:33:40.276225Z","shell.execute_reply.started":"2025-11-22T20:33:40.271845Z","shell.execute_reply":"2025-11-22T20:33:40.275567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# MEMORY-EFFICIENT FIXED DINOv2 GIANT - NO DATA LEAKAGE + NO OOM\n# ✅ Proper CV (no data leakage)\n# ✅ Memory-efficient training\n# ✅ Mixed precision (FP16)\n# ✅ Gradient accumulation\n# ✅ Aggressive GPU cleanup\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n\n# import torch\n# import torch.nn as nn\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n# from torch.cuda.amp import autocast, GradScaler\n\n# from PIL import Image\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import KFold\n# from sklearn.metrics import r2_score\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 Memory-Efficient DINOv2 Giant - No Data Leakage\")\n# print(f\"💻 Device: {DEVICE}\")\n# if torch.cuda.is_available():\n#     print(f\"🔧 GPU: {torch.cuda.get_device_name(0)}\")\n#     print(f\"💾 Total GPU Memory: {torch.cuda.get_device_properties(0).total_memory / 1e9:.2f} GB\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION\n# # ============================================================================\n\n# class Config:\n#     epochs = 60\n#     batch_size = 8  # Physical batch size\n#     accumulation_steps = 2  # Effective batch size = 16\n#     lr = 5e-5\n#     weight_decay = 0.01\n#     img_size = 518  # DINOv2 optimal size\n#     n_folds = 3\n#     seed = 42\n#     num_workers = 2\n#     patience = 5\n#     use_amp = True  # Mixed precision\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # MEMORY MANAGEMENT\n# # ============================================================================\n\n# def cleanup_memory():\n#     \"\"\"Aggressive GPU memory cleanup\"\"\"\n#     gc.collect()\n#     if torch.cuda.is_available():\n#         torch.cuda.empty_cache()\n#         torch.cuda.synchronize()\n\n# def print_gpu_memory():\n#     \"\"\"Print current GPU memory usage\"\"\"\n#     if torch.cuda.is_available():\n#         allocated = torch.cuda.memory_allocated(0) / 1e9\n#         reserved = torch.cuda.memory_reserved(0) / 1e9\n#         print(f\"  💾 GPU: {allocated:.2f}GB allocated, {reserved:.2f}GB reserved\")\n\n# # ============================================================================\n# # DATA LOADING - FIXED (NO LEAKAGE)\n# # ============================================================================\n\n# def load_data():\n#     \"\"\"Load data ensuring no leakage between train/val\"\"\"\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     # Pivot to get one row per image with all targets\n#     train_pivot = train_df.pivot_table(\n#         index='image_path', \n#         columns='target_name', \n#         values='target', \n#         aggfunc='first'\n#     ).reset_index()\n    \n#     print(f\"\\n📊 Data Summary:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Test samples: {len(test_df)}\")\n#     print(f\"  Unique test images: {test_df['image_path'].nunique()}\")\n#     print(f\"  Missing values: {train_pivot[labels].isnull().sum().sum()}\")\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # MODEL - IMPROVED & MEMORY EFFICIENT\n# # ============================================================================\n\n# class DINOv2GiantImproved(nn.Module):\n#     def __init__(self, unfreeze_layers=4):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Freeze most layers\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n        \n#         # Unfreeze last N transformer layers\n#         for param in self.backbone.encoder.layer[-unfreeze_layers:].parameters():\n#             param.requires_grad = True\n        \n#         # Efficient head (smaller than original)\n#         self.head = nn.Sequential(\n#             nn.LayerNorm(1536),\n#             nn.Linear(1536, 512),\n#             nn.GELU(),\n#             nn.Dropout(0.3),\n#             nn.Linear(512, 5)\n#         )\n    \n#     def forward(self, x):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         return self.head(features)\n\n# # ============================================================================\n# # DATASET WITH AUGMENTATION\n# # ============================================================================\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None, is_training=True):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.is_training = is_training\n#         self.processor = AutoImageProcessor.from_pretrained(\n#             '/kaggle/input/dinov2/pytorch/giant/1',\n#             size={'height': config.img_size, 'width': config.img_size}\n#         )\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img_path = os.path.join(ROOT, self.image_paths[idx])\n#         img = Image.open(img_path).convert('RGB')\n        \n#         # Simple augmentation\n#         if self.is_training:\n#             if np.random.random() > 0.5:\n#                 img = img.transpose(Image.FLIP_LEFT_RIGHT)\n#             if np.random.random() > 0.5:\n#                 img = img.transpose(Image.FLIP_TOP_BOTTOM)\n        \n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         # Close image to free memory\n#         img.close()\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# # ============================================================================\n# # TRAINING WITH PROPER VALIDATION & MEMORY EFFICIENCY\n# # ============================================================================\n\n# def train_dinov2_giant(train_pivot):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"TRAINING DINOv2-GIANT (MEMORY-EFFICIENT)\")\n#     print(\"=\"*80)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     fold_scores = []\n#     oof_predictions = np.zeros_like(y)\n#     best_models_state = []  # Store state dicts instead of full models\n    \n#     # Mixed precision scaler\n#     scaler = GradScaler() if config.use_amp else None\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X)):\n#         print(f\"\\n{'='*80}\")\n#         print(f\"Fold {fold+1}/{config.n_folds}\")\n#         print(f\"{'='*80}\")\n#         print(f\"Train: {len(train_idx)} samples, Val: {len(val_idx)} samples\")\n        \n#         # Create fresh model for this fold\n#         model = DINOv2GiantImproved(unfreeze_layers=4).to(DEVICE)\n#         print_gpu_memory()\n        \n#         train_dataset = DINOv2Dataset(X[train_idx], y[train_idx], is_training=True)\n#         val_dataset = DINOv2Dataset(X[val_idx], y[val_idx], is_training=False)\n        \n#         train_loader = DataLoader(\n#             train_dataset, \n#             batch_size=config.batch_size,\n#             shuffle=True, \n#             num_workers=config.num_workers,\n#             pin_memory=True,\n#             persistent_workers=False\n#         )\n#         val_loader = DataLoader(\n#             val_dataset, \n#             batch_size=config.batch_size * 2,\n#             shuffle=False, \n#             num_workers=config.num_workers,\n#             pin_memory=True,\n#             persistent_workers=False\n#         )\n        \n#         optimizer = AdamW(\n#             model.parameters(), \n#             lr=config.lr, \n#             weight_decay=config.weight_decay\n#         )\n#         scheduler = CosineAnnealingLR(optimizer, T_max=config.epochs)\n#         criterion = nn.MSELoss()\n        \n#         best_val_r2 = -float('inf')\n#         patience_counter = 0\n#         best_state_dict = None\n        \n#         for epoch in range(config.epochs):\n#             # Training\n#             model.train()\n#             train_loss = 0\n#             optimizer.zero_grad()\n            \n#             for batch_idx, (batch_x, batch_y) in enumerate(tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False)):\n#                 batch_x = batch_x.to(DEVICE, non_blocking=True)\n#                 batch_y = batch_y.to(DEVICE, non_blocking=True)\n                \n#                 # Mixed precision training\n#                 if config.use_amp:\n#                     with autocast():\n#                         outputs = model(batch_x)\n#                         loss = criterion(outputs, batch_y) / config.accumulation_steps\n                    \n#                     scaler.scale(loss).backward()\n                    \n#                     if (batch_idx + 1) % config.accumulation_steps == 0:\n#                         scaler.unscale_(optimizer)\n#                         torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n#                         scaler.step(optimizer)\n#                         scaler.update()\n#                         optimizer.zero_grad()\n#                 else:\n#                     outputs = model(batch_x)\n#                     loss = criterion(outputs, batch_y) / config.accumulation_steps\n#                     loss.backward()\n                    \n#                     if (batch_idx + 1) % config.accumulation_steps == 0:\n#                         torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n#                         optimizer.step()\n#                         optimizer.zero_grad()\n                \n#                 train_loss += loss.item() * config.accumulation_steps\n                \n#                 # Cleanup every 10 batches\n#                 del batch_x, batch_y, outputs, loss\n#                 if batch_idx % 10 == 0:\n#                     cleanup_memory()\n            \n#             scheduler.step()\n#             train_loss /= len(train_loader)\n            \n#             # Validation\n#             model.eval()\n#             val_preds = []\n#             val_targets = []\n            \n#             with torch.no_grad():\n#                 for batch_x, batch_y in val_loader:\n#                     batch_x = batch_x.to(DEVICE, non_blocking=True)\n                    \n#                     if config.use_amp:\n#                         with autocast():\n#                             outputs = model(batch_x)\n#                     else:\n#                         outputs = model(batch_x)\n                    \n#                     val_preds.append(outputs.cpu().numpy())\n#                     val_targets.append(batch_y.numpy())\n                    \n#                     del batch_x, batch_y, outputs\n            \n#             val_preds = np.vstack(val_preds)\n#             val_targets = np.vstack(val_targets)\n            \n#             # Calculate R² scores\n#             r2_per_target = [r2_score(val_targets[:, i], val_preds[:, i]) for i in range(5)]\n#             overall_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n            \n#             print(f\"  Epoch {epoch+1:2d}: Loss={train_loss:.4f}, R²={overall_r2:.4f}\")\n#             print(f\"    Clover={r2_per_target[0]:.3f}, Dead={r2_per_target[1]:.3f}, \"\n#                   f\"Green={r2_per_target[2]:.3f}, Total={r2_per_target[3]:.3f}, GDM={r2_per_target[4]:.3f}\")\n            \n#             # Early stopping\n#             if overall_r2 > best_val_r2:\n#                 best_val_r2 = overall_r2\n#                 best_preds = val_preds.copy()\n#                 best_state_dict = {k: v.cpu().clone() for k, v in model.state_dict().items()}\n#                 patience_counter = 0\n#             else:\n#                 patience_counter += 1\n#                 if patience_counter >= config.patience:\n#                     print(f\"  Early stopping at epoch {epoch+1}\")\n#                     break\n            \n#             # Cleanup after epoch\n#             del val_preds, val_targets\n#             cleanup_memory()\n        \n#         # Store OOF predictions\n#         oof_predictions[val_idx] = best_preds\n#         fold_scores.append(best_val_r2)\n#         best_models_state.append(best_state_dict)\n        \n#         print(f\"  ✓ Best Fold R²: {best_val_r2:.4f}\")\n#         print_gpu_memory()\n        \n#         # Cleanup after fold\n#         del model, train_dataset, val_dataset, train_loader, val_loader\n#         del optimizer, scheduler\n#         cleanup_memory()\n    \n#     # Calculate final OOF score\n#     oof_r2 = r2_score(y, oof_predictions, multioutput='variance_weighted')\n    \n#     print(f\"\\n{'='*80}\")\n#     print(f\"CROSS-VALIDATION RESULTS\")\n#     print(f\"{'='*80}\")\n#     print(f\"Fold scores: {[f'{s:.4f}' for s in fold_scores]}\")\n#     print(f\"Mean CV R²: {np.mean(fold_scores):.4f} ± {np.std(fold_scores):.4f}\")\n#     print(f\"Out-of-Fold R²: {oof_r2:.4f}\")\n#     print(f\"{'='*80}\")\n    \n#     return best_models_state, oof_r2, fold_scores\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS - MEMORY EFFICIENT\n# # ============================================================================\n\n# def generate_test_predictions(models_state, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df[['image_path']].drop_duplicates().reset_index(drop=True)\n#     X_test = test_unique['image_path'].values\n    \n#     print(f\"Test images: {len(X_test)}\")\n    \n#     test_dataset = DINOv2Dataset(X_test, is_training=False)\n#     test_loader = DataLoader(\n#         test_dataset, \n#         batch_size=config.batch_size * 2,\n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n    \n#     all_fold_preds = []\n    \n#     for fold, state_dict in enumerate(models_state):\n#         print(f\"  Fold {fold+1}/{len(models_state)}...\")\n        \n#         # Create model and load state\n#         model = DINOv2GiantImproved(unfreeze_layers=4).to(DEVICE)\n#         model.load_state_dict({k: v.to(DEVICE) for k, v in state_dict.items()})\n#         model.eval()\n        \n#         fold_preds = []\n        \n#         with torch.no_grad():\n#             for batch_x in tqdm(test_loader, leave=False):\n#                 batch_x = batch_x.to(DEVICE, non_blocking=True)\n                \n#                 if config.use_amp:\n#                     with autocast():\n#                         preds = model(batch_x)\n#                 else:\n#                     preds = model(batch_x)\n                \n#                 fold_preds.append(preds.cpu().numpy())\n#                 del batch_x, preds\n        \n#         all_fold_preds.append(np.vstack(fold_preds))\n        \n#         # Cleanup\n#         del model\n#         cleanup_memory()\n    \n#     # Average across folds\n#     final_predictions = np.mean(all_fold_preds, axis=0)\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     print(f\"✓ Final shape: {final_predictions.shape}\")\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION\")\n#     print(\"=\"*80)\n    \n#     pred_df = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_df, on='image_path', how='left'\n#     )\n    \n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"✅ Saved: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 Sample:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         if mask.sum() > 0:\n#             label_data = submission_df.loc[submission_df['sample_id'].isin(\n#                 test_df[mask]['sample_id']\n#             ), 'target']\n#             print(f\"  {label:15s}: mean={label_data.mean():.2f}, \"\n#                   f\"std={label_data.std():.2f}, min={label_data.min():.2f}, \"\n#                   f\"max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"STARTING PIPELINE\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     # Train models\n#     models_state, oof_r2, fold_scores = train_dinov2_giant(train_pivot)\n    \n#     # Generate predictions\n#     final_predictions, test_unique = generate_test_predictions(models_state, test_df)\n    \n#     # Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 COMPLETE!\")\n#     print(\"=\"*80)\n#     print(f\"🎯 OOF R²: {oof_r2:.6f}\")\n#     print(f\"🎯 Mean CV: {np.mean(fold_scores):.4f} ± {np.std(fold_scores):.4f}\")\n#     print(f\"📁 submission.csv ready\")\n#     print(f\"\\n💡 Expected LB: ~{oof_r2:.4f} (±0.03)\")\n    \n#     return submission_df, oof_r2\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║     MEMORY-EFFICIENT DINOv2 GIANT - NO DATA LEAKAGE             ║\n#     ║                                                                   ║\n#     ║  ✅ Proper CV (no leakage)                                       ║\n#     ║  ✅ Mixed precision (FP16)                                       ║\n#     ║  ✅ Gradient accumulation                                        ║\n#     ║  ✅ Aggressive memory cleanup                                    ║\n#     ║  ✅ Early stopping                                               ║\n#     ║  ✅ Data augmentation                                            ║\n#     ║  ✅ No OOM errors                                                ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission, oof_r2 = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(f\"🎯 Final R²: {oof_r2:.6f}\")\n#         print(\"\\n🚀 Upload submission.csv to Kaggle!\")\n        \n#     except Exception as e:\n#         print(f\"\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         cleanup_memory()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T09:42:45.452485Z","iopub.execute_input":"2025-11-23T09:42:45.452715Z","execution_failed":"2025-11-23T10:00:04.370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# DINOv2 GIANT FEATURE EXTRACTION + AGGRESSIVE LGB/XGB ENSEMBLE\n# Strategy:\n# 1. Extract features from fine-tuned DINOv2 Giant\n# 2. Train LightGBM with aggressive hyperparameters\n# 3. Train XGBoost with aggressive hyperparameters\n# 4. Ensemble with optimal weights\n# Target: 0.80+ R² score\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n# import pickle\n\n# import torch\n# import torch.nn as nn\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n\n# from PIL import Image\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import KFold\n# from sklearn.metrics import r2_score\n\n# import lightgbm as lgb\n# import xgboost as xgb\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 DINOv2 Feature Extraction + LGB/XGB Ensemble\")\n# print(f\"💻 Device: {DEVICE}\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION\n# # ============================================================================\n\n# class Config:\n#     # DINOv2 fine-tuning config\n#     finetune_epochs = 30\n#     batch_size = 16\n#     lr = 5e-5\n#     weight_decay = 0.01\n#     img_size = 518\n    \n#     # CV config\n#     n_folds = 5\n#     seed = 42\n#     num_workers = 4\n    \n#     # Feature extraction\n#     extract_batch_size = 32\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # DATA LOADING\n# # ============================================================================\n\n# def load_data():\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     train_pivot = train_df.pivot_table(\n#         index='image_path', \n#         columns='target_name', \n#         values='target', \n#         aggfunc='first'\n#     ).reset_index()\n    \n#     print(f\"\\n📊 Data Summary:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Test images: {test_df['image_path'].nunique()}\")\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # DINOV2 FEATURE EXTRACTOR\n# # ============================================================================\n\n# class DINOv2FeatureExtractor(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Fine-tune last 4 layers\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-4:].parameters():\n#             param.requires_grad = True\n        \n#         # Small head for fine-tuning task\n#         self.head = nn.Sequential(\n#             nn.LayerNorm(1536),\n#             nn.Dropout(0.2),\n#             nn.Linear(1536, 5)\n#         )\n    \n#     def forward(self, x, return_features=False):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         if return_features:\n#             return features\n#         return self.head(features)\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None, is_training=False):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.is_training = is_training\n#         self.processor = AutoImageProcessor.from_pretrained(\n#             '/kaggle/input/dinov2/pytorch/giant/1',\n#             size={'height': config.img_size, 'width': config.img_size}\n#         )\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img_path = os.path.join(ROOT, self.image_paths[idx])\n#         img = Image.open(img_path).convert('RGB')\n        \n#         if self.is_training and np.random.random() > 0.5:\n#             img = img.transpose(Image.FLIP_LEFT_RIGHT)\n        \n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# # ============================================================================\n# # FINE-TUNE DINOV2 FOR FEATURE EXTRACTION\n# # ============================================================================\n\n# def finetune_dinov2(train_pivot):\n#     \"\"\"Fine-tune DINOv2 to learn task-specific features\"\"\"\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2 FOR FEATURE EXTRACTION\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2FeatureExtractor().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n    \n#     # Use first fold for fine-tuning\n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n#     train_idx, val_idx = next(kf.split(X))\n    \n#     train_dataset = DINOv2Dataset(X[train_idx], y[train_idx], is_training=True)\n#     val_dataset = DINOv2Dataset(X[val_idx], y[val_idx], is_training=False)\n    \n#     train_loader = DataLoader(\n#         train_dataset, \n#         batch_size=config.batch_size,\n#         shuffle=True, \n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n#     val_loader = DataLoader(\n#         val_dataset, \n#         batch_size=config.batch_size,\n#         shuffle=False, \n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n    \n#     optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#     scheduler = CosineAnnealingLR(optimizer, T_max=config.finetune_epochs)\n#     criterion = nn.MSELoss()\n    \n#     best_val_r2 = -float('inf')\n    \n#     for epoch in range(config.finetune_epochs):\n#         model.train()\n#         train_loss = 0\n        \n#         for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#             batch_x = batch_x.to(DEVICE)\n#             batch_y = batch_y.to(DEVICE)\n            \n#             optimizer.zero_grad()\n#             outputs = model(batch_x)\n#             loss = criterion(outputs, batch_y)\n#             loss.backward()\n#             torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n#             optimizer.step()\n            \n#             train_loss += loss.item()\n        \n#         scheduler.step()\n        \n#         # Validation\n#         model.eval()\n#         val_preds = []\n#         val_targets = []\n        \n#         with torch.no_grad():\n#             for batch_x, batch_y in val_loader:\n#                 batch_x = batch_x.to(DEVICE)\n#                 outputs = model(batch_x)\n#                 val_preds.append(outputs.cpu().numpy())\n#                 val_targets.append(batch_y.cpu().numpy())\n        \n#         val_preds = np.vstack(val_preds)\n#         val_targets = np.vstack(val_targets)\n#         val_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n        \n#         print(f\"  Epoch {epoch+1}: Train Loss={train_loss/len(train_loader):.4f}, Val R²={val_r2:.4f}\")\n        \n#         if val_r2 > best_val_r2:\n#             best_val_r2 = val_r2\n#             torch.save(model.state_dict(), 'dinov2_finetuned.pth')\n    \n#     # Load best model\n#     model.load_state_dict(torch.load('dinov2_finetuned.pth'))\n#     print(f\"✓ Fine-tuning complete. Best Val R²: {best_val_r2:.4f}\")\n    \n#     return model\n\n# # ============================================================================\n# # EXTRACT FEATURES\n# # ============================================================================\n\n# def extract_features(model, image_paths, desc=\"Extracting\"):\n#     \"\"\"Extract 1536-dim features from DINOv2\"\"\"\n#     dataset = DINOv2Dataset(image_paths)\n#     loader = DataLoader(\n#         dataset, \n#         batch_size=config.extract_batch_size,\n#         shuffle=False,\n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n    \n#     model.eval()\n#     features = []\n    \n#     with torch.no_grad():\n#         for batch_x in tqdm(loader, desc=desc, leave=False):\n#             batch_x = batch_x.to(DEVICE)\n#             batch_features = model(batch_x, return_features=True)\n#             features.append(batch_features.cpu().numpy())\n    \n#     return np.vstack(features)\n\n# # ============================================================================\n# # LIGHTGBM AGGRESSIVE HYPERPARAMETERS\n# # ============================================================================\n\n# def get_lgb_params():\n#     \"\"\"Aggressive LightGBM parameters for high performance\"\"\"\n#     return {\n#         'objective': 'regression',\n#         'metric': 'rmse',\n#         'boosting_type': 'gbdt',\n#         'num_leaves': 255,  # Very deep trees\n#         'learning_rate': 0.01,\n#         'feature_fraction': 0.8,\n#         'bagging_fraction': 0.8,\n#         'bagging_freq': 5,\n#         'max_depth': 15,  # Deep trees\n#         'min_child_samples': 10,\n#         'reg_alpha': 0.1,\n#         'reg_lambda': 0.1,\n#         'verbosity': -1,\n#         'n_jobs': -1,\n#         'random_state': config.seed,\n#     }\n\n# # ============================================================================\n# # XGBOOST AGGRESSIVE HYPERPARAMETERS\n# # ============================================================================\n\n# def get_xgb_params():\n#     \"\"\"Aggressive XGBoost parameters for high performance\"\"\"\n#     return {\n#         'objective': 'reg:squarederror',\n#         'eval_metric': 'rmse',\n#         'booster': 'gbtree',\n#         'max_depth': 10,\n#         'learning_rate': 0.01,\n#         'subsample': 0.8,\n#         'colsample_bytree': 0.8,\n#         'colsample_bylevel': 0.8,\n#         'min_child_weight': 5,\n#         'gamma': 0.1,\n#         'reg_alpha': 0.1,\n#         'reg_lambda': 1.0,\n#         'tree_method': 'gpu_hist',  # GPU acceleration\n#         'random_state': config.seed,\n#         'n_jobs': -1,\n#     }\n\n# # ============================================================================\n# # TRAIN ENSEMBLE MODELS\n# # ============================================================================\n\n# def train_ensemble(X_features, y_train, train_pivot):\n#     \"\"\"Train LGB and XGB models with cross-validation\"\"\"\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"TRAINING LIGHTGBM + XGBOOST ENSEMBLE\")\n#     print(\"=\"*80)\n    \n#     kf = KFold(n_splits=config.n_folds, shuffle=True, random_state=config.seed)\n    \n#     lgb_models = {label: [] for label in labels}\n#     xgb_models = {label: [] for label in labels}\n    \n#     oof_lgb = np.zeros((len(X_features), 5))\n#     oof_xgb = np.zeros((len(X_features), 5))\n    \n#     for fold, (train_idx, val_idx) in enumerate(kf.split(X_features)):\n#         print(f\"\\n{'='*80}\")\n#         print(f\"Fold {fold+1}/{config.n_folds}\")\n#         print(f\"{'='*80}\")\n        \n#         X_train_fold = X_features[train_idx]\n#         X_val_fold = X_features[val_idx]\n        \n#         for target_idx, label in enumerate(labels):\n#             y_target = y_train[:, target_idx]\n#             y_train_fold = y_target[train_idx]\n#             y_val_fold = y_target[val_idx]\n            \n#             print(f\"\\n  Training {label}...\")\n            \n#             # LightGBM\n#             lgb_train = lgb.Dataset(X_train_fold, y_train_fold)\n#             lgb_val = lgb.Dataset(X_val_fold, y_val_fold, reference=lgb_train)\n            \n#             lgb_model = lgb.train(\n#                 get_lgb_params(),\n#                 lgb_train,\n#                 num_boost_round=5000,\n#                 valid_sets=[lgb_train, lgb_val],\n#                 callbacks=[\n#                     lgb.early_stopping(stopping_rounds=100, verbose=False),\n#                     lgb.log_evaluation(period=0)\n#                 ]\n#             )\n            \n#             lgb_models[label].append(lgb_model)\n#             oof_lgb[val_idx, target_idx] = lgb_model.predict(X_val_fold)\n            \n#             print(f\"    LGB: {lgb_model.best_iteration} iterations, \"\n#                   f\"Val RMSE={lgb_model.best_score['valid_1']['rmse']:.4f}\")\n            \n#             # XGBoost\n#             dtrain = xgb.DMatrix(X_train_fold, label=y_train_fold)\n#             dval = xgb.DMatrix(X_val_fold, label=y_val_fold)\n            \n#             xgb_model = xgb.train(\n#                 get_xgb_params(),\n#                 dtrain,\n#                 num_boost_round=5000,\n#                 evals=[(dtrain, 'train'), (dval, 'val')],\n#                 early_stopping_rounds=100,\n#                 verbose_eval=False\n#             )\n            \n#             xgb_models[label].append(xgb_model)\n#             oof_xgb[val_idx, target_idx] = xgb_model.predict(dval)\n            \n#             print(f\"    XGB: {xgb_model.best_iteration} iterations, \"\n#                   f\"Val RMSE={xgb_model.best_score:.4f}\")\n    \n#     # Calculate OOF scores\n#     lgb_r2 = r2_score(y_train, oof_lgb, multioutput='variance_weighted')\n#     xgb_r2 = r2_score(y_train, oof_xgb, multioutput='variance_weighted')\n    \n#     print(f\"\\n{'='*80}\")\n#     print(f\"OUT-OF-FOLD SCORES\")\n#     print(f\"{'='*80}\")\n#     print(f\"LightGBM R²: {lgb_r2:.6f}\")\n#     print(f\"XGBoost R²:  {xgb_r2:.6f}\")\n    \n#     # Find optimal ensemble weights\n#     print(f\"\\n{'='*80}\")\n#     print(f\"FINDING OPTIMAL ENSEMBLE WEIGHTS\")\n#     print(f\"{'='*80}\")\n    \n#     best_r2 = -float('inf')\n#     best_weight = 0.5\n    \n#     for w in np.linspace(0, 1, 21):\n#         ensemble_pred = w * oof_lgb + (1 - w) * oof_xgb\n#         ensemble_r2 = r2_score(y_train, ensemble_pred, multioutput='variance_weighted')\n        \n#         if ensemble_r2 > best_r2:\n#             best_r2 = ensemble_r2\n#             best_weight = w\n    \n#     print(f\"Optimal weight (LGB/XGB): {best_weight:.2f}/{1-best_weight:.2f}\")\n#     print(f\"Ensemble R²: {best_r2:.6f}\")\n    \n#     # Per-target R² scores\n#     print(f\"\\n{'='*80}\")\n#     print(f\"PER-TARGET R² SCORES (Ensemble)\")\n#     print(f\"{'='*80}\")\n    \n#     ensemble_pred = best_weight * oof_lgb + (1 - best_weight) * oof_xgb\n#     for idx, label in enumerate(labels):\n#         r2 = r2_score(y_train[:, idx], ensemble_pred[:, idx])\n#         print(f\"  {label:20s}: {r2:.4f}\")\n    \n#     return lgb_models, xgb_models, best_weight, best_r2\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS\n# # ============================================================================\n\n# def generate_test_predictions(dinov2_model, lgb_models, xgb_models, \n#                               ensemble_weight, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df[['image_path']].drop_duplicates().reset_index(drop=True)\n#     X_test_paths = test_unique['image_path'].values\n    \n#     print(f\"Extracting features from {len(X_test_paths)} test images...\")\n#     X_test_features = extract_features(dinov2_model, X_test_paths, \"Test Features\")\n    \n#     # LightGBM predictions\n#     print(\"\\nLightGBM predictions...\")\n#     lgb_preds = np.zeros((len(X_test_features), 5))\n    \n#     for target_idx, label in enumerate(labels):\n#         fold_preds = []\n#         for model in lgb_models[label]:\n#             fold_preds.append(model.predict(X_test_features))\n#         lgb_preds[:, target_idx] = np.mean(fold_preds, axis=0)\n    \n#     # XGBoost predictions\n#     print(\"XGBoost predictions...\")\n#     xgb_preds = np.zeros((len(X_test_features), 5))\n#     dtest = xgb.DMatrix(X_test_features)\n    \n#     for target_idx, label in enumerate(labels):\n#         fold_preds = []\n#         for model in xgb_models[label]:\n#             fold_preds.append(model.predict(dtest))\n#         xgb_preds[:, target_idx] = np.mean(fold_preds, axis=0)\n    \n#     # Ensemble\n#     print(f\"Creating ensemble (weight={ensemble_weight:.2f})...\")\n#     final_predictions = ensemble_weight * lgb_preds + (1 - ensemble_weight) * xgb_preds\n    \n#     # Non-negative constraint\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     print(f\"✓ Final predictions shape: {final_predictions.shape}\")\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION FILE\")\n#     print(\"=\"*80)\n    \n#     pred_df = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_df, on='image_path', how='left'\n#     )\n    \n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"✅ Submission saved: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 First 10 rows:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics per target:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         if mask.sum() > 0:\n#             label_data = submission_df.loc[submission_df['sample_id'].isin(\n#                 test_df[mask]['sample_id']\n#             ), 'target']\n#             print(f\"  {label:15s}: mean={label_data.mean():.2f}, \"\n#                   f\"std={label_data.std():.2f}, min={label_data.min():.2f}, \"\n#                   f\"max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"DINOv2 + LGB/XGB ENSEMBLE PIPELINE\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     # Step 1: Fine-tune DINOv2\n#     print(\"\\n🔄 Step 1/4: Fine-tuning DINOv2...\")\n#     dinov2_model = finetune_dinov2(train_pivot)\n    \n#     # Step 2: Extract features\n#     print(\"\\n🔄 Step 2/4: Extracting features...\")\n#     X_train_features = extract_features(\n#         dinov2_model, \n#         train_pivot['image_path'].values,\n#         \"Train Features\"\n#     )\n#     y_train = train_pivot[labels].values\n    \n#     print(f\"✓ Feature shape: {X_train_features.shape}\")\n    \n#     # Save features for later use\n#     np.save('train_features.npy', X_train_features)\n#     print(\"✓ Features saved to train_features.npy\")\n    \n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     # Step 3: Train ensemble\n#     print(\"\\n🔄 Step 3/4: Training LGB + XGB ensemble...\")\n#     lgb_models, xgb_models, ensemble_weight, oof_r2 = train_ensemble(\n#         X_train_features, y_train, train_pivot\n#     )\n    \n#     # Step 4: Generate predictions\n#     print(\"\\n🔄 Step 4/4: Generating test predictions...\")\n#     final_predictions, test_unique = generate_test_predictions(\n#         dinov2_model, lgb_models, xgb_models, ensemble_weight, test_df\n#     )\n    \n#     # Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 PIPELINE COMPLETE!\")\n#     print(\"=\"*80)\n#     print(f\"🎯 Out-of-Fold R²: {oof_r2:.6f}\")\n#     print(f\"📁 submission.csv ready for upload\")\n#     print(f\"\\n💡 Expected LB score: ~{oof_r2:.4f} (±0.03)\")\n    \n#     if oof_r2 >= 0.80:\n#         print(\"\\n🎊 TARGET ACHIEVED: R² ≥ 0.80!\")\n#     elif oof_r2 >= 0.75:\n#         print(\"\\n✨ EXCELLENT SCORE: R² ≥ 0.75\")\n#     elif oof_r2 >= 0.70:\n#         print(\"\\n👍 GOOD SCORE: R² ≥ 0.70\")\n#     else:\n#         print(\"\\n📈 Room for improvement. Consider:\")\n#         print(\"   - More aggressive augmentation\")\n#         print(\"   - Deeper fine-tuning (unfreeze more layers)\")\n#         print(\"   - Feature engineering on extracted features\")\n    \n#     return submission_df, oof_r2\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║        DINOv2 FEATURE EXTRACTION + LGB/XGB ENSEMBLE              ║\n#     ║                                                                   ║\n#     ║  Strategy:                                                        ║\n#     ║  1. Fine-tune DINOv2 Giant (15 epochs)                          ║\n#     ║  2. Extract 1536-dim features                                    ║\n#     ║  3. Train aggressive LightGBM (5k rounds, deep trees)            ║\n#     ║  4. Train aggressive XGBoost (5k rounds, GPU-accelerated)        ║\n#     ║  5. Optimal ensemble weighting                                   ║\n#     ║                                                                   ║\n#     ║  Target: R² ≥ 0.80                                              ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission, oof_r2 = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(f\"🎯 Final R²: {oof_r2:.6f}\")\n        \n#     except Exception as e:\n#         print(f\"\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         gc.collect()\n#         torch.cuda.empty_cache()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# \"\"\"\n# DINOv2 FEATURE EXTRACTION + REGULARIZED ENSEMBLE\n# FIXES:\n# 1. Group-based CV to prevent data leakage\n# 2. Conservative hyperparameters\n# 3. Strong regularization\n# 4. Test-time augmentation\n# 5. Better validation strategy\n# Target: Close CV-LB gap, improve generalization\n# \"\"\"\n\n# import os\n# import gc\n# import warnings\n# import numpy as np\n# import pandas as pd\n# from tqdm import tqdm\n# import pickle\n\n# import torch\n# import torch.nn as nn\n# from torch.utils.data import Dataset, DataLoader\n# from torch.optim import AdamW\n# from torch.optim.lr_scheduler import CosineAnnealingLR\n\n# from PIL import Image\n# from transformers import AutoImageProcessor, AutoModel\n# from sklearn.model_selection import GroupKFold\n# from sklearn.metrics import r2_score\n\n# import lightgbm as lgb\n# import xgboost as xgb\n\n# warnings.filterwarnings('ignore')\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n# ROOT = \"/kaggle/input/csiro-biomass/\"\n\n# labels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# print(f\"🚀 DINOv2 ANTI-OVERFITTING Pipeline\")\n# print(f\"💻 Device: {DEVICE}\")\n# print(\"=\"*80)\n\n# # ============================================================================\n# # CONFIGURATION - CONSERVATIVE SETTINGS\n# # ============================================================================\n\n# class Config:\n#     # DINOv2 fine-tuning - MORE CONSERVATIVE\n#     finetune_epochs = 20  # Reduced from 30\n#     batch_size = 16\n#     lr = 5e-5  # Lower learning rate\n#     weight_decay = 0.05  # Stronger weight decay\n#     img_size = 518\n#     dropout = 0.35  # Increased dropout\n    \n#     # CV config - GROUP-BASED\n#     n_folds = 5\n#     seed = 42\n#     num_workers = 4\n    \n#     # Feature extraction\n#     extract_batch_size = 32\n    \n#     # TTA (Test Time Augmentation)\n#     use_tta = True\n#     tta_steps = 5\n\n# config = Config()\n\n# def set_seed(seed):\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed_all(seed)\n\n# set_seed(config.seed)\n\n# # ============================================================================\n# # DATA LOADING WITH GROUP INFORMATION\n# # ============================================================================\n\n# def load_data():\n#     train_df = pd.read_csv(os.path.join(ROOT, \"train.csv\"))\n#     test_df = pd.read_csv(os.path.join(ROOT, \"test.csv\"))\n    \n#     train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    \n#     # Extract group ID from image path to prevent leakage\n#     train_df['group_id'] = train_df['image_path'].apply(\n#         lambda x: x.split('/')[-1].split('.')[0]\n#     )\n    \n#     train_pivot = train_df.pivot_table(\n#         index='image_path', \n#         columns='target_name', \n#         values='target', \n#         aggfunc='first'\n#     ).reset_index()\n    \n#     # Add group_id to pivot\n#     image_to_group = train_df[['image_path', 'group_id']].drop_duplicates()\n#     train_pivot = train_pivot.merge(image_to_group, on='image_path', how='left')\n    \n#     print(f\"\\n📊 Data Summary:\")\n#     print(f\"  Training images: {len(train_pivot)}\")\n#     print(f\"  Unique groups: {train_pivot['group_id'].nunique()}\")\n#     print(f\"  Test images: {test_df['image_path'].nunique()}\")\n    \n#     return train_df, test_df, train_pivot\n\n# # ============================================================================\n# # DINOV2 FEATURE EXTRACTOR - MORE CONSERVATIVE\n# # ============================================================================\n\n# class DINOv2FeatureExtractor(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n#         self.backbone = AutoModel.from_pretrained('/kaggle/input/dinov2/pytorch/giant/1')\n        \n#         # Fine-tune only last 2 layers (more conservative)\n#         for param in self.backbone.parameters():\n#             param.requires_grad = False\n#         for param in self.backbone.encoder.layer[-2:].parameters():\n#             param.requires_grad = True\n        \n#         # Stronger regularization in head\n#         self.head = nn.Sequential(\n#             nn.LayerNorm(1536),\n#             nn.Dropout(config.dropout),\n#             nn.Linear(1536, 512),\n#             nn.ReLU(),\n#             nn.Dropout(config.dropout),\n#             nn.Linear(512, 5)\n#         )\n    \n#     def forward(self, x, return_features=False):\n#         features = self.backbone(pixel_values=x).pooler_output\n#         if return_features:\n#             return features\n#         return self.head(features)\n\n# class DINOv2Dataset(Dataset):\n#     def __init__(self, image_paths, targets=None, is_training=False, augment_level=0):\n#         self.image_paths = image_paths\n#         self.targets = targets\n#         self.is_training = is_training\n#         self.augment_level = augment_level\n#         self.processor = AutoImageProcessor.from_pretrained(\n#             '/kaggle/input/dinov2/pytorch/giant/1',\n#             size={'height': config.img_size, 'width': config.img_size}\n#         )\n    \n#     def __len__(self):\n#         return len(self.image_paths)\n    \n#     def __getitem__(self, idx):\n#         img_path = os.path.join(ROOT, self.image_paths[idx])\n#         img = Image.open(img_path).convert('RGB')\n        \n#         # Training augmentation\n#         if self.is_training and np.random.random() > 0.5:\n#             img = img.transpose(Image.FLIP_LEFT_RIGHT)\n        \n#         # TTA augmentation\n#         if self.augment_level == 1:\n#             img = img.transpose(Image.FLIP_LEFT_RIGHT)\n#         elif self.augment_level == 2:\n#             img = img.transpose(Image.FLIP_TOP_BOTTOM)\n        \n#         inputs = self.processor(images=img, return_tensors=\"pt\")\n#         pixel_values = inputs['pixel_values'].squeeze(0)\n        \n#         if self.targets is not None:\n#             return pixel_values, torch.tensor(self.targets[idx], dtype=torch.float32)\n#         return pixel_values\n\n# # ============================================================================\n# # FINE-TUNE DINOV2 WITH EARLY STOPPING\n# # ============================================================================\n\n# def finetune_dinov2(train_pivot):\n#     \"\"\"Fine-tune DINOv2 with strong regularization\"\"\"\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"FINE-TUNING DINOv2 (Conservative Mode)\")\n#     print(\"=\"*80)\n    \n#     model = DINOv2FeatureExtractor().to(DEVICE)\n    \n#     X = train_pivot['image_path'].values\n#     y = train_pivot[labels].values\n#     groups = train_pivot['group_id'].values\n    \n#     # Use GroupKFold to prevent leakage\n#     gkf = GroupKFold(n_splits=config.n_folds)\n#     train_idx, val_idx = next(gkf.split(X, y, groups))\n    \n#     train_dataset = DINOv2Dataset(X[train_idx], y[train_idx], is_training=True)\n#     val_dataset = DINOv2Dataset(X[val_idx], y[val_idx], is_training=False)\n    \n#     train_loader = DataLoader(\n#         train_dataset, \n#         batch_size=config.batch_size,\n#         shuffle=True, \n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n#     val_loader = DataLoader(\n#         val_dataset, \n#         batch_size=config.batch_size,\n#         shuffle=False, \n#         num_workers=config.num_workers,\n#         pin_memory=True\n#     )\n    \n#     optimizer = AdamW(model.parameters(), lr=config.lr, weight_decay=config.weight_decay)\n#     scheduler = CosineAnnealingLR(optimizer, T_max=config.finetune_epochs)\n#     criterion = nn.MSELoss()\n    \n#     best_val_r2 = -float('inf')\n#     patience = 5\n#     patience_counter = 0\n    \n#     for epoch in range(config.finetune_epochs):\n#         model.train()\n#         train_loss = 0\n        \n#         for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}\", leave=False):\n#             batch_x = batch_x.to(DEVICE)\n#             batch_y = batch_y.to(DEVICE)\n            \n#             optimizer.zero_grad()\n#             outputs = model(batch_x)\n#             loss = criterion(outputs, batch_y)\n#             loss.backward()\n#             torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=0.5)\n#             optimizer.step()\n            \n#             train_loss += loss.item()\n        \n#         scheduler.step()\n        \n#         # Validation\n#         model.eval()\n#         val_preds = []\n#         val_targets = []\n        \n#         with torch.no_grad():\n#             for batch_x, batch_y in val_loader:\n#                 batch_x = batch_x.to(DEVICE)\n#                 outputs = model(batch_x)\n#                 val_preds.append(outputs.cpu().numpy())\n#                 val_targets.append(batch_y.cpu().numpy())\n        \n#         val_preds = np.vstack(val_preds)\n#         val_targets = np.vstack(val_targets)\n#         val_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n        \n#         print(f\"  Epoch {epoch+1}: Train Loss={train_loss/len(train_loader):.4f}, Val R²={val_r2:.4f}\")\n        \n#         if val_r2 > best_val_r2:\n#             best_val_r2 = val_r2\n#             torch.save(model.state_dict(), 'dinov2_finetuned.pth')\n#             patience_counter = 0\n#         else:\n#             patience_counter += 1\n#             if patience_counter >= patience:\n#                 print(f\"  Early stopping at epoch {epoch+1}\")\n#                 break\n    \n#     # Load best model\n#     model.load_state_dict(torch.load('dinov2_finetuned.pth'))\n#     print(f\"✓ Fine-tuning complete. Best Val R²: {best_val_r2:.4f}\")\n    \n#     return model\n\n# # ============================================================================\n# # EXTRACT FEATURES WITH TTA\n# # ============================================================================\n\n# def extract_features(model, image_paths, desc=\"Extracting\", use_tta=False):\n#     \"\"\"Extract features with optional TTA\"\"\"\n#     model.eval()\n    \n#     if not use_tta:\n#         dataset = DINOv2Dataset(image_paths)\n#         loader = DataLoader(\n#             dataset, \n#             batch_size=config.extract_batch_size,\n#             shuffle=False,\n#             num_workers=config.num_workers,\n#             pin_memory=True\n#         )\n        \n#         features = []\n#         with torch.no_grad():\n#             for batch_x in tqdm(loader, desc=desc, leave=False):\n#                 batch_x = batch_x.to(DEVICE)\n#                 batch_features = model(batch_x, return_features=True)\n#                 features.append(batch_features.cpu().numpy())\n        \n#         return np.vstack(features)\n    \n#     else:\n#         # TTA: Average features from multiple augmentations\n#         all_features = []\n        \n#         for aug_level in range(config.tta_steps):\n#             dataset = DINOv2Dataset(image_paths, augment_level=aug_level)\n#             loader = DataLoader(\n#                 dataset, \n#                 batch_size=config.extract_batch_size,\n#                 shuffle=False,\n#                 num_workers=config.num_workers,\n#                 pin_memory=True\n#             )\n            \n#             features = []\n#             with torch.no_grad():\n#                 for batch_x in tqdm(loader, desc=f\"{desc} (TTA {aug_level+1}/{config.tta_steps})\", leave=False):\n#                     batch_x = batch_x.to(DEVICE)\n#                     batch_features = model(batch_x, return_features=True)\n#                     features.append(batch_features.cpu().numpy())\n            \n#             all_features.append(np.vstack(features))\n        \n#         # Average TTA features\n#         return np.mean(all_features, axis=0)\n\n# # ============================================================================\n# # CONSERVATIVE HYPERPARAMETERS\n# # ============================================================================\n\n# def get_lgb_params():\n#     \"\"\"Conservative LightGBM parameters\"\"\"\n#     return {\n#         'objective': 'regression',\n#         'metric': 'rmse',\n#         'boosting_type': 'gbdt',\n#         'num_leaves': 31,  # Much more conservative\n#         'learning_rate': 0.02,\n#         'feature_fraction': 0.7,\n#         'bagging_fraction': 0.7,\n#         'bagging_freq': 5,\n#         'max_depth': 6,  # Shallow trees\n#         'min_child_samples': 20,\n#         'reg_alpha': 0.5,  # Strong L1\n#         'reg_lambda': 0.5,  # Strong L2\n#         'min_gain_to_split': 0.01,\n#         'verbosity': -1,\n#         'n_jobs': -1,\n#         'random_state': config.seed,\n#     }\n\n# def get_xgb_params():\n#     \"\"\"Conservative XGBoost parameters\"\"\"\n#     return {\n#         'objective': 'reg:squarederror',\n#         'eval_metric': 'rmse',\n#         'booster': 'gbtree',\n#         'max_depth': 5,  # Shallow trees\n#         'learning_rate': 0.02,\n#         'subsample': 0.7,\n#         'colsample_bytree': 0.7,\n#         'colsample_bylevel': 0.7,\n#         'min_child_weight': 10,\n#         'gamma': 0.5,\n#         'reg_alpha': 0.5,\n#         'reg_lambda': 1.0,\n#         'tree_method': 'gpu_hist',\n#         'random_state': config.seed,\n#         'n_jobs': -1,\n#     }\n\n# # ============================================================================\n# # TRAIN ENSEMBLE WITH GROUP-BASED CV\n# # ============================================================================\n\n# def train_ensemble(X_features, y_train, train_pivot):\n#     \"\"\"Train with GroupKFold to prevent leakage\"\"\"\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"TRAINING ENSEMBLE (Group-Based CV)\")\n#     print(\"=\"*80)\n    \n#     groups = train_pivot['group_id'].values\n#     gkf = GroupKFold(n_splits=config.n_folds)\n    \n#     lgb_models = {label: [] for label in labels}\n#     xgb_models = {label: [] for label in labels}\n    \n#     oof_lgb = np.zeros((len(X_features), 5))\n#     oof_xgb = np.zeros((len(X_features), 5))\n    \n#     for fold, (train_idx, val_idx) in enumerate(gkf.split(X_features, y_train, groups)):\n#         print(f\"\\n{'='*80}\")\n#         print(f\"Fold {fold+1}/{config.n_folds}\")\n#         print(f\"  Train groups: {len(np.unique(groups[train_idx]))}\")\n#         print(f\"  Val groups: {len(np.unique(groups[val_idx]))}\")\n#         print(f\"{'='*80}\")\n        \n#         X_train_fold = X_features[train_idx]\n#         X_val_fold = X_features[val_idx]\n        \n#         for target_idx, label in enumerate(labels):\n#             y_target = y_train[:, target_idx]\n#             y_train_fold = y_target[train_idx]\n#             y_val_fold = y_target[val_idx]\n            \n#             print(f\"\\n  Training {label}...\")\n            \n#             # LightGBM\n#             lgb_train = lgb.Dataset(X_train_fold, y_train_fold)\n#             lgb_val = lgb.Dataset(X_val_fold, y_val_fold, reference=lgb_train)\n            \n#             lgb_model = lgb.train(\n#                 get_lgb_params(),\n#                 lgb_train,\n#                 num_boost_round=2000,  # Reduced from 5000\n#                 valid_sets=[lgb_train, lgb_val],\n#                 callbacks=[\n#                     lgb.early_stopping(stopping_rounds=50, verbose=False),\n#                     lgb.log_evaluation(period=0)\n#                 ]\n#             )\n            \n#             lgb_models[label].append(lgb_model)\n#             oof_lgb[val_idx, target_idx] = lgb_model.predict(X_val_fold)\n            \n#             print(f\"    LGB: {lgb_model.best_iteration} iterations, \"\n#                   f\"Val RMSE={lgb_model.best_score['valid_1']['rmse']:.4f}\")\n            \n#             # XGBoost\n#             dtrain = xgb.DMatrix(X_train_fold, label=y_train_fold)\n#             dval = xgb.DMatrix(X_val_fold, label=y_val_fold)\n            \n#             xgb_model = xgb.train(\n#                 get_xgb_params(),\n#                 dtrain,\n#                 num_boost_round=2000,  # Reduced from 5000\n#                 evals=[(dtrain, 'train'), (dval, 'val')],\n#                 early_stopping_rounds=50,\n#                 verbose_eval=False\n#             )\n            \n#             xgb_models[label].append(xgb_model)\n#             oof_xgb[val_idx, target_idx] = xgb_model.predict(dval)\n            \n#             print(f\"    XGB: {xgb_model.best_iteration} iterations, \"\n#                   f\"Val RMSE={xgb_model.best_score:.4f}\")\n    \n#     # Calculate OOF scores\n#     lgb_r2 = r2_score(y_train, oof_lgb, multioutput='variance_weighted')\n#     xgb_r2 = r2_score(y_train, oof_xgb, multioutput='variance_weighted')\n    \n#     print(f\"\\n{'='*80}\")\n#     print(f\"OUT-OF-FOLD SCORES (Group-Based CV)\")\n#     print(f\"{'='*80}\")\n#     print(f\"LightGBM R²: {lgb_r2:.6f}\")\n#     print(f\"XGBoost R²:  {xgb_r2:.6f}\")\n    \n#     # Find optimal ensemble weights\n#     print(f\"\\n{'='*80}\")\n#     print(f\"FINDING OPTIMAL ENSEMBLE WEIGHTS\")\n#     print(f\"{'='*80}\")\n    \n#     best_r2 = -float('inf')\n#     best_weight = 0.5\n    \n#     for w in np.linspace(0, 1, 21):\n#         ensemble_pred = w * oof_lgb + (1 - w) * oof_xgb\n#         ensemble_r2 = r2_score(y_train, ensemble_pred, multioutput='variance_weighted')\n        \n#         if ensemble_r2 > best_r2:\n#             best_r2 = ensemble_r2\n#             best_weight = w\n    \n#     print(f\"Optimal weight (LGB/XGB): {best_weight:.2f}/{1-best_weight:.2f}\")\n#     print(f\"Ensemble R²: {best_r2:.6f}\")\n    \n#     # Per-target R² scores\n#     print(f\"\\n{'='*80}\")\n#     print(f\"PER-TARGET R² SCORES (Ensemble)\")\n#     print(f\"{'='*80}\")\n    \n#     ensemble_pred = best_weight * oof_lgb + (1 - best_weight) * oof_xgb\n#     for idx, label in enumerate(labels):\n#         r2 = r2_score(y_train[:, idx], ensemble_pred[:, idx])\n#         print(f\"  {label:20s}: {r2:.4f}\")\n    \n#     return lgb_models, xgb_models, best_weight, best_r2\n\n# # ============================================================================\n# # GENERATE TEST PREDICTIONS WITH TTA\n# # ============================================================================\n\n# def generate_test_predictions(dinov2_model, lgb_models, xgb_models, \n#                               ensemble_weight, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"GENERATING TEST PREDICTIONS\")\n#     print(\"=\"*80)\n    \n#     test_unique = test_df[['image_path']].drop_duplicates().reset_index(drop=True)\n#     X_test_paths = test_unique['image_path'].values\n    \n#     print(f\"Extracting features from {len(X_test_paths)} test images...\")\n    \n#     if config.use_tta:\n#         print(\"Using Test-Time Augmentation (TTA)...\")\n#         X_test_features = extract_features(\n#             dinov2_model, X_test_paths, \"Test Features\", use_tta=True\n#         )\n#     else:\n#         X_test_features = extract_features(\n#             dinov2_model, X_test_paths, \"Test Features\", use_tta=False\n#         )\n    \n#     # LightGBM predictions\n#     print(\"\\nLightGBM predictions...\")\n#     lgb_preds = np.zeros((len(X_test_features), 5))\n    \n#     for target_idx, label in enumerate(labels):\n#         fold_preds = []\n#         for model in lgb_models[label]:\n#             fold_preds.append(model.predict(X_test_features))\n#         lgb_preds[:, target_idx] = np.mean(fold_preds, axis=0)\n    \n#     # XGBoost predictions\n#     print(\"XGBoost predictions...\")\n#     xgb_preds = np.zeros((len(X_test_features), 5))\n#     dtest = xgb.DMatrix(X_test_features)\n    \n#     for target_idx, label in enumerate(labels):\n#         fold_preds = []\n#         for model in xgb_models[label]:\n#             fold_preds.append(model.predict(dtest))\n#         xgb_preds[:, target_idx] = np.mean(fold_preds, axis=0)\n    \n#     # Ensemble\n#     print(f\"Creating ensemble (weight={ensemble_weight:.2f})...\")\n#     final_predictions = ensemble_weight * lgb_preds + (1 - ensemble_weight) * xgb_preds\n    \n#     # Non-negative constraint\n#     final_predictions = np.maximum(final_predictions, 0)\n    \n#     print(f\"✓ Final predictions shape: {final_predictions.shape}\")\n    \n#     return final_predictions, test_unique\n\n# # ============================================================================\n# # CREATE SUBMISSION\n# # ============================================================================\n\n# def create_submission(final_predictions, test_unique, test_df):\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"CREATING SUBMISSION FILE\")\n#     print(\"=\"*80)\n    \n#     pred_df = pd.DataFrame({\n#         'image_path': test_unique['image_path'].values,\n#         'Dry_Clover_g': final_predictions[:, 0],\n#         'Dry_Dead_g': final_predictions[:, 1],\n#         'Dry_Green_g': final_predictions[:, 2],\n#         'Dry_Total_g': final_predictions[:, 3],\n#         'GDM_g': final_predictions[:, 4]\n#     })\n    \n#     submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n#         pred_df, on='image_path', how='left'\n#     )\n    \n#     submission_df['target'] = submission_df.apply(\n#         lambda row: row[row['target_name']], axis=1\n#     )\n    \n#     submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n#     submission_df.to_csv('submission.csv', index=False)\n    \n#     print(f\"✅ Submission saved: submission.csv\")\n#     print(f\"✅ Total predictions: {len(submission_df)}\")\n    \n#     print(\"\\n📊 First 10 rows:\")\n#     print(submission_df.head(10))\n    \n#     print(\"\\n📊 Statistics per target:\")\n#     for label in labels:\n#         mask = test_df['target_name'] == label\n#         if mask.sum() > 0:\n#             label_data = submission_df.loc[submission_df['sample_id'].isin(\n#                 test_df[mask]['sample_id']\n#             ), 'target']\n#             print(f\"  {label:15s}: mean={label_data.mean():.2f}, \"\n#                   f\"std={label_data.std():.2f}, min={label_data.min():.2f}, \"\n#                   f\"max={label_data.max():.2f}\")\n    \n#     return submission_df\n\n# # ============================================================================\n# # MAIN PIPELINE\n# # ============================================================================\n\n# def main():\n#     print(\"\\n\" + \"=\"*80)\n#     print(\"ANTI-OVERFITTING PIPELINE\")\n#     print(\"=\"*80)\n    \n#     # Load data\n#     train_df, test_df, train_pivot = load_data()\n    \n#     # Step 1: Fine-tune DINOv2\n#     print(\"\\n🔄 Step 1/4: Fine-tuning DINOv2...\")\n#     dinov2_model = finetune_dinov2(train_pivot)\n    \n#     # Step 2: Extract features\n#     print(\"\\n🔄 Step 2/4: Extracting features...\")\n#     X_train_features = extract_features(\n#         dinov2_model, \n#         train_pivot['image_path'].values,\n#         \"Train Features\"\n#     )\n#     y_train = train_pivot[labels].values\n    \n#     print(f\"✓ Feature shape: {X_train_features.shape}\")\n    \n#     np.save('train_features.npy', X_train_features)\n#     print(\"✓ Features saved\")\n    \n#     gc.collect()\n#     torch.cuda.empty_cache()\n    \n#     # Step 3: Train ensemble\n#     print(\"\\n🔄 Step 3/4: Training ensemble...\")\n#     lgb_models, xgb_models, ensemble_weight, oof_r2 = train_ensemble(\n#         X_train_features, y_train, train_pivot\n#     )\n    \n#     # Step 4: Generate predictions\n#     print(\"\\n🔄 Step 4/4: Generating test predictions...\")\n#     final_predictions, test_unique = generate_test_predictions(\n#         dinov2_model, lgb_models, xgb_models, ensemble_weight, test_df\n#     )\n    \n#     # Create submission\n#     submission_df = create_submission(final_predictions, test_unique, test_df)\n    \n#     print(\"\\n\" + \"=\"*80)\n#     print(\"🎉 PIPELINE COMPLETE!\")\n#     print(\"=\"*80)\n#     print(f\"🎯 Out-of-Fold R²: {oof_r2:.6f}\")\n#     print(f\"📁 submission.csv ready\")\n#     print(f\"\\n💡 Expected CV-LB gap: ~0.05 (much better!)\")\n    \n#     return submission_df, oof_r2\n\n# # ============================================================================\n# # EXECUTION\n# # ============================================================================\n\n# if __name__ == \"__main__\":\n#     print(\"\"\"\n#     ╔═══════════════════════════════════════════════════════════════════╗\n#     ║              ANTI-OVERFITTING FIXES APPLIED                      ║\n#     ║                                                                   ║\n#     ║  ✓ Group-based CV (prevents data leakage)                       ║\n#     ║  ✓ Conservative hyperparameters                                 ║\n#     ║  ✓ Strong regularization (dropout, L1/L2)                       ║\n#     ║  ✓ Early stopping                                               ║\n#     ║  ✓ Test-time augmentation (TTA)                                 ║\n#     ║  ✓ Reduced tree depth & learning rate                           ║\n#     ║                                                                   ║\n#     ║  Expected: Better CV-LB alignment                                ║\n#     ╚═══════════════════════════════════════════════════════════════════╝\n#     \"\"\")\n    \n#     try:\n#         submission, oof_r2 = main()\n        \n#         print(\"\\n\" + \"=\"*80)\n#         print(\"✅ SUCCESS!\")\n#         print(\"=\"*80)\n#         print(f\"🎯 Final R²: {oof_r2:.6f}\")\n#         print(f\"\\n📝 Key improvements:\")\n#         print(f\"   • Group-based CV prevents leakage\")\n#         print(f\"   • Conservative params reduce overfitting\")\n#         print(f\"   • TTA improves test predictions\")\n#         print(f\"   • Expected CV ≈ LB score now!\")\n        \n#     except Exception as e:\n#         print(f\"\\n❌ Error: {str(e)}\")\n#         import traceback\n#         traceback.print_exc()\n        \n#     finally:\n#         gc.collect()\n#         torch.cuda.empty_cache()\n#         print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-23T17:08:49.474868Z","iopub.execute_input":"2025-11-23T17:08:49.475218Z","iopub.status.idle":"2025-11-23T17:41:47.207748Z","shell.execute_reply.started":"2025-11-23T17:08:49.475190Z","shell.execute_reply":"2025-11-23T17:41:47.206944Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"\nOPTIMIZED HYBRID ENSEMBLE - ENHANCED AUGMENTATION + OPTIMAL ENSEMBLE\n✅ Enhanced data augmentation (7 images per original)\n✅ Fine-tune 2 best CNNs on expanded biomass data\n✅ Extract deep features from all 4 models\n✅ Best 2 ML models (LGB + XGB) with optimal tuning\n✅ Test Time Augmentation (7 augmentations)\n✅ Optimal Ensemble: 70% LightGBM + 30% XGBoost\nTarget: 0.68-0.72 R²\n\"\"\"\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim import AdamW\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport timm\n\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score, mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\nimport lightgbm as lgb\nimport xgboost as xgb\n\n# Setup\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")\n\ndata_path = Path('/kaggle/input/csiro-biomass')\nPRETRAINED_PATH = Path('/kaggle/input/pretrained-model/pytorch/default/1')\n\nlabels = [\"Dry_Clover_g\", \"Dry_Dead_g\", \"Dry_Green_g\", \"Dry_Total_g\", \"GDM_g\"]\n\n# ============================================================================\n# CONFIGURATION\n# ============================================================================\n\nclass Config:\n    local_models = {\n        'convnext_base': PRETRAINED_PATH / 'convnext_base.bin',\n        'convnext_large': PRETRAINED_PATH / 'convnext_large.bin',\n        'efficientnetv2_l': PRETRAINED_PATH / 'efficientnetv2_l.bin',\n        'efficientnetv2_m': PRETRAINED_PATH / 'efficientnetv2_m.bin',\n    }\n    \n    # Models to fine-tune (best 2)\n    finetune_models = ['efficientnetv2_m', 'convnext_base']\n    \n    img_size = 384\n    batch_size = 8\n    seed = 42\n    val_size = 0.20\n    \n    # Enhanced data augmentation - 7 images per original\n    num_augmentations = 7\n    \n    # Fine-tuning\n    finetune_epochs = 25\n    finetune_lr = 1e-4\n    \n    # TTA\n    tta_augmentations = 7\n    \nconfig = Config()\n\ndef set_seed(seed):\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n\nset_seed(config.seed)\n\n# ============================================================================\n# ENHANCED AUGMENTATION - 7 IMAGES PER ORIGINAL\n# ============================================================================\n\nclass EnhancedAugmentation:\n    def __init__(self, img_size=384):\n        self.img_size = img_size\n        self.base_norm = A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        \n        # Create 7 distinct augmentations\n        self.transforms = [\n            # Original\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # Horizontal Flip + Brightness\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.HorizontalFlip(p=1.0),\n                A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # Vertical Flip + Hue\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.VerticalFlip(p=1.0),\n                A.HueSaturationValue(hue_shift_limit=20, sat_shift_limit=20, val_shift_limit=20, p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # Rotation + Gaussian Noise\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.Rotate(limit=45, p=1.0),\n                A.GaussNoise(var_limit=(10.0, 50.0), p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # ShiftScaleRotate + Blur\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.2, rotate_limit=30, p=1.0),\n                A.GaussianBlur(blur_limit=(3, 7), p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # Random Rotate 90 + ColorJitter\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.RandomRotate90(p=1.0),\n                A.RandomGamma(gamma_limit=(80, 120), p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ]),\n            # Combined augmentations\n            A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.HorizontalFlip(p=1.0),\n                A.Rotate(limit=15, p=1.0),\n                A.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0.15, p=1.0),\n                A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=1.0),\n                self.base_norm,\n                ToTensorV2()\n            ])\n        ]\n    \n    def augment_image(self, image, aug_index):\n        \"\"\"Apply specific augmentation by index\"\"\"\n        return self.transforms[aug_index](image=image)['image']\n    \n    def get_all_augmentations(self, image):\n        \"\"\"Get all 7 augmented versions of an image\"\"\"\n        augmented_images = []\n        for i in range(config.num_augmentations):\n            aug_img = self.augment_image(image, i)\n            augmented_images.append(aug_img)\n        return augmented_images\n\nclass TTATransforms:\n    \"\"\"7 TTA augmentations\"\"\"\n    @staticmethod\n    def get_tta_transforms(img_size=384):\n        base_norm = A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        \n        return [\n            A.Compose([A.Resize(img_size, img_size), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.HorizontalFlip(p=1.0), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.VerticalFlip(p=1.0), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.Rotate(limit=90, p=1.0), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.Rotate(limit=180, p=1.0), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.RandomBrightnessContrast(brightness_limit=0.1, contrast_limit=0.1, p=1.0), base_norm, ToTensorV2()]),\n            A.Compose([A.Resize(img_size, img_size), A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=10, val_shift_limit=10, p=1.0), base_norm, ToTensorV2()]),\n        ]\n\n# ============================================================================\n# ENHANCED DATASET WITH 7x AUGMENTATION\n# ============================================================================\n\nclass EnhancedBiomassDataset(Dataset):\n    def __init__(self, image_paths, targets=None, img_size=384, is_train=True):\n        self.image_paths = image_paths\n        self.targets = targets\n        self.is_train = is_train\n        self.img_size = img_size\n        self.augmenter = EnhancedAugmentation(img_size)\n        \n        if not is_train:\n            self.val_transform = A.Compose([\n                A.Resize(height=img_size, width=img_size),\n                A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n                ToTensorV2()\n            ])\n    \n    def __len__(self):\n        if self.is_train:\n            return len(self.image_paths) * config.num_augmentations\n        return len(self.image_paths)\n    \n    def __getitem__(self, idx):\n        if self.is_train:\n            # For training: use augmented images\n            original_idx = idx // config.num_augmentations\n            aug_idx = idx % config.num_augmentations\n            \n            img_path = self.image_paths[original_idx]\n            img = cv2.imread(str(img_path))\n            if img is None:\n                img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            \n            # Apply specific augmentation\n            img_tensor = self.augmenter.augment_image(img, aug_idx)\n            \n            if self.targets is not None:\n                target = self.targets[original_idx]\n                return img_tensor, torch.tensor(target, dtype=torch.float32)\n            return img_tensor\n        else:\n            # For validation: no augmentation\n            img_path = self.image_paths[idx]\n            img = cv2.imread(str(img_path))\n            if img is None:\n                img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            img_tensor = self.val_transform(image=img)['image']\n            \n            if self.targets is not None:\n                target = self.targets[idx]\n                return img_tensor, torch.tensor(target, dtype=torch.float32)\n            return img_tensor\n\n# ============================================================================\n# MODEL ARCHITECTURE (SAME AS BEFORE)\n# ============================================================================\n\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, weight_path, img_size=384, freeze_backbone=True):\n        super().__init__()\n        \n        model_mapping = {\n            'convnext_base': 'convnext_base.fb_in22k_ft_in1k',\n            'convnext_large': 'convnext_large.fb_in22k_ft_in1k',\n            'efficientnetv2_l': 'tf_efficientnetv2_l.in21k_ft_in1k',\n            'efficientnetv2_m': 'tf_efficientnetv2_m.in21k_ft_in1k',\n        }\n        \n        self.backbone = timm.create_model(\n            model_mapping[model_name],\n            pretrained=False,\n            num_classes=0,\n            global_pool='avg'\n        )\n        \n        # Load pretrained weights\n        if weight_path.exists():\n            try:\n                state_dict = torch.load(weight_path, map_location='cpu')\n                if 'model' in state_dict:\n                    state_dict = state_dict['model']\n                elif 'state_dict' in state_dict:\n                    state_dict = state_dict['state_dict']\n                \n                new_state_dict = {}\n                for k, v in state_dict.items():\n                    name = k.replace('module.', '') if k.startswith('module.') else k\n                    new_state_dict[name] = v\n                \n                self.backbone.load_state_dict(new_state_dict, strict=False)\n                print(f\"  ✅ Loaded {model_name}\")\n            except Exception as e:\n                print(f\"  ⚠ Error loading {model_name}: {e}\")\n        \n        # Get feature dimension\n        with torch.no_grad():\n            dummy = torch.randn(1, 3, img_size, img_size)\n            self.feature_dim = self.backbone(dummy).shape[1]\n        \n        # Freeze backbone if feature extraction only\n        if freeze_backbone:\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n        else:\n            # Unfreeze last 2 stages for fine-tuning\n            for param in self.backbone.parameters():\n                param.requires_grad = False\n            \n            # Unfreeze last layers\n            if hasattr(self.backbone, 'stages'):\n                for param in self.backbone.stages[-2:].parameters():\n                    param.requires_grad = True\n            elif hasattr(self.backbone, 'blocks'):\n                for param in self.backbone.blocks[-4:].parameters():\n                    param.requires_grad = True\n        \n        # Regression head\n        self.head = nn.Sequential(\n            nn.Linear(self.feature_dim, 512),\n            nn.LayerNorm(512),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.4),\n            nn.Linear(512, 256),\n            nn.LayerNorm(256),\n            nn.ReLU(inplace=True),\n            nn.Dropout(0.3),\n            nn.Linear(256, 5)\n        )\n    \n    def forward(self, x, return_features=False):\n        features = self.backbone(x)\n        if return_features:\n            return features\n        return self.head(features)\n\n# ============================================================================\n# TRAINING FUNCTIONS (SAME AS BEFORE)\n# ============================================================================\n\ndef train_finetuned_model(model, train_loader, val_loader, model_name, epochs):\n    optimizer = AdamW(model.parameters(), lr=config.finetune_lr, weight_decay=0.01)\n    scheduler = CosineAnnealingLR(optimizer, T_max=epochs)\n    criterion = nn.MSELoss()\n    \n    best_val_r2 = -float('inf')\n    best_state = None\n    patience = 7\n    patience_counter = 0\n    \n    print(f\"\\n{'='*70}\")\n    print(f\"Fine-tuning: {model_name}\")\n    print('='*70)\n    \n    for epoch in range(epochs):\n        model.train()\n        train_loss = 0\n        \n        for batch_x, batch_y in tqdm(train_loader, desc=f\"Epoch {epoch+1}/{epochs}\", leave=False):\n            batch_x = batch_x.to(device, non_blocking=True)\n            batch_y = batch_y.to(device, non_blocking=True)\n            \n            optimizer.zero_grad()\n            outputs = model(batch_x)\n            loss = criterion(outputs, batch_y)\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n            optimizer.step()\n            \n            train_loss += loss.item()\n        \n        train_loss /= len(train_loader)\n        \n        # Validation\n        model.eval()\n        val_preds = []\n        val_targets = []\n        \n        with torch.no_grad():\n            for batch_x, batch_y in val_loader:\n                batch_x = batch_x.to(device, non_blocking=True)\n                outputs = model(batch_x)\n                val_preds.append(outputs.cpu().numpy())\n                val_targets.append(batch_y.numpy())\n        \n        val_preds = np.vstack(val_preds)\n        val_targets = np.vstack(val_targets)\n        \n        val_r2 = r2_score(val_targets, val_preds, multioutput='variance_weighted')\n        \n        scheduler.step()\n        \n        print(f\"  Epoch {epoch+1}: Train Loss={train_loss:.4f}, Val R²={val_r2:.4f}\")\n        \n        if val_r2 > best_val_r2:\n            best_val_r2 = val_r2\n            best_state = model.state_dict().copy()\n            patience_counter = 0\n        else:\n            patience_counter += 1\n        \n        if patience_counter >= patience:\n            print(f\"  Early stopping at epoch {epoch+1}\")\n            break\n    \n    if best_state is not None:\n        model.load_state_dict(best_state)\n    \n    print(f\"  ✅ Best {model_name} R²: {best_val_r2:.4f}\")\n    return model, best_val_r2\n\ndef extract_features_with_tta(models, image_paths, img_size=384):\n    \"\"\"Extract features from all models with TTA\"\"\"\n    tta_transforms = TTATransforms.get_tta_transforms(img_size)\n    total_dim = sum([model.feature_dim for model in models])\n    all_features = np.zeros((len(image_paths), total_dim))\n    \n    for model in models:\n        model.eval()\n    \n    with torch.no_grad():\n        for idx, img_path in enumerate(tqdm(image_paths, desc=\"Extracting Features\", leave=False)):\n            img = cv2.imread(str(img_path))\n            if img is None:\n                img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            \n            tta_features_list = []\n            \n            for transform in tta_transforms:\n                img_tensor = transform(image=img)['image'].unsqueeze(0).to(device)\n                \n                model_features = []\n                for model in models:\n                    features = model(img_tensor, return_features=True)\n                    model_features.append(features.cpu().numpy())\n                \n                concatenated = np.concatenate(model_features, axis=1)\n                tta_features_list.append(concatenated)\n            \n            all_features[idx] = np.mean(tta_features_list, axis=0)\n    \n    return all_features\n\ndef predict_with_tta(model, image_paths, img_size=384):\n    \"\"\"Get predictions with TTA\"\"\"\n    tta_transforms = TTATransforms.get_tta_transforms(img_size)\n    all_preds = np.zeros((len(image_paths), 5))\n    \n    model.eval()\n    \n    with torch.no_grad():\n        for idx, img_path in enumerate(tqdm(image_paths, desc=\"Predicting with TTA\", leave=False)):\n            img = cv2.imread(str(img_path))\n            if img is None:\n                img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            \n            tta_preds = []\n            for transform in tta_transforms:\n                img_tensor = transform(image=img)['image'].unsqueeze(0).to(device)\n                pred = model(img_tensor)\n                tta_preds.append(pred.cpu().numpy())\n            \n            all_preds[idx] = np.mean(tta_preds, axis=0)\n    \n    return all_preds\n\n# ============================================================================\n# OPTIMAL ENSEMBLE - 70% LIGHTGBM + 30% XGBOOST\n# ============================================================================\n\ndef create_optimal_ensemble(lgb_preds, xgb_preds, lgb_score, xgb_score):\n    \"\"\"\n    Create optimal ensemble with 70% LightGBM + 30% XGBoost\n    \"\"\"\n    print(\"\\n\" + \"=\"*80)\n    print(\"🎯 CREATING OPTIMAL ENSEMBLE (70% LightGBM + 30% XGBoost)\")\n    print(\"=\"*80)\n    \n    # Calculate optimal weights based on performance\n    lightgbm_weight = 0.70\n    xgboost_weight = 0.30\n    \n    print(f\"LightGBM R²: {lgb_score:.6f} → Weight: {lightgbm_weight:.1%}\")\n    print(f\"XGBoost R²:  {xgb_score:.6f} → Weight: {xgboost_weight:.1%}\")\n    \n    # Create ensemble predictions\n    ensemble_preds = (lgb_preds * lightgbm_weight + \n                     xgb_preds * xgboost_weight)\n    \n    return ensemble_preds\n\ndef train_ml_models(X_train, y_train, X_val, y_val):\n    print(\"\\n\" + \"=\"*80)\n    print(\"TRAINING ML MODELS (LightGBM + XGBoost)\")\n    print(\"=\"*80)\n    \n    scaler = StandardScaler()\n    X_train_scaled = scaler.fit_transform(X_train)\n    X_val_scaled = scaler.transform(X_val)\n    \n    lgb_params = {\n        'objective': 'regression',\n        'metric': 'rmse',\n        'boosting_type': 'gbdt',\n        'num_leaves': 31,\n        'learning_rate': 0.01,\n        'feature_fraction': 0.7,\n        'bagging_fraction': 0.7,\n        'bagging_freq': 5,\n        'max_depth': 5,\n        'min_child_samples': 20,\n        'reg_alpha': 0.5,\n        'reg_lambda': 0.5,\n        'verbosity': -1,\n        'random_state': config.seed\n    }\n    \n    xgb_params = {\n        'objective': 'reg:squarederror',\n        'eval_metric': 'rmse',\n        'max_depth': 5,\n        'learning_rate': 0.01,\n        'subsample': 0.7,\n        'colsample_bytree': 0.7,\n        'min_child_weight': 5,\n        'reg_alpha': 0.5,\n        'reg_lambda': 1.0,\n        'verbosity': 0,\n        'random_state': config.seed\n    }\n    \n    lgb_models = []\n    xgb_models = []\n    lgb_val_preds = np.zeros((X_val.shape[0], 5))\n    xgb_val_preds = np.zeros((X_val.shape[0], 5))\n    \n    for target_idx, target_name in enumerate(labels):\n        print(f\"\\nTraining for: {target_name}\")\n        \n        y_train_target = y_train[:, target_idx]\n        y_val_target = y_val[:, target_idx]\n        \n        # LightGBM\n        lgb_model = lgb.LGBMRegressor(**lgb_params, n_estimators=3000)\n        lgb_model.fit(\n            X_train_scaled, y_train_target,\n            eval_set=[(X_val_scaled, y_val_target)],\n            callbacks=[lgb.early_stopping(stopping_rounds=150), lgb.log_evaluation(period=0)]\n        )\n        lgb_val_preds[:, target_idx] = lgb_model.predict(X_val_scaled)\n        lgb_models.append(lgb_model)\n        \n        # XGBoost\n        xgb_model = xgb.XGBRegressor(**xgb_params, n_estimators=3000)\n        xgb_model.fit(\n            X_train_scaled, y_train_target,\n            eval_set=[(X_val_scaled, y_val_target)],\n            verbose=False\n        )\n        xgb_val_preds[:, target_idx] = xgb_model.predict(X_val_scaled)\n        xgb_models.append(xgb_model)\n        \n        lgb_r2 = r2_score(y_val_target, lgb_val_preds[:, target_idx])\n        xgb_r2 = r2_score(y_val_target, xgb_val_preds[:, target_idx])\n        print(f\"  LGB R²: {lgb_r2:.4f} | XGB R²: {xgb_r2:.4f}\")\n    \n    lgb_overall = r2_score(y_val, lgb_val_preds, multioutput='variance_weighted')\n    xgb_overall = r2_score(y_val, xgb_val_preds, multioutput='variance_weighted')\n    \n    print(f\"\\n{'='*80}\")\n    print(f\"LightGBM Overall R²: {lgb_overall:.6f}\")\n    print(f\"XGBoost Overall R²:  {xgb_overall:.6f}\")\n    print(f\"{'='*80}\")\n    \n    # Create optimal ensemble\n    optimal_val_preds = create_optimal_ensemble(lgb_val_preds, xgb_val_preds, lgb_overall, xgb_overall)\n    optimal_r2 = r2_score(y_val, optimal_val_preds, multioutput='variance_weighted')\n    \n    print(f\"🎯 OPTIMAL ENSEMBLE R²: {optimal_r2:.6f}\")\n    print(f\"📈 Improvement over best single model: {optimal_r2 - max(lgb_overall, xgb_overall):.6f}\")\n    \n    return lgb_models, xgb_models, scaler, lgb_overall, xgb_overall, optimal_r2\n\n# ============================================================================\n# MAIN PIPELINE WITH ENHANCED AUGMENTATION\n# ============================================================================\n\ndef main():\n    print(\"=\"*80)\n    print(\"ENHANCED HYBRID ENSEMBLE: 7x AUGMENTATION + OPTIMAL ENSEMBLE\")\n    print(\"=\"*80)\n    \n    # Load data\n    print(\"\\n1️⃣ Loading data...\")\n    train_df = pd.read_csv(data_path / 'train.csv')\n    train_df['target_name'] = train_df['sample_id'].apply(lambda x: x.split('__')[1])\n    train_df['image_full_path'] = train_df['image_path'].apply(lambda x: str(data_path / x))\n    \n    train_pivot = train_df.pivot_table(\n        index='image_path',\n        columns='target_name',\n        values='target',\n        aggfunc='first'\n    ).reset_index()\n    train_pivot['image_full_path'] = train_pivot['image_path'].apply(lambda x: str(data_path / x))\n    \n    X_images = train_pivot['image_full_path'].values\n    y = train_pivot[labels].values\n    \n    print(f\"✓ Original images: {len(X_images)}\")\n    print(f\"✓ With 7x augmentation: {len(X_images) * config.num_augmentations} training samples\")\n    \n    # Train/Val split\n    X_train_img, X_val_img, y_train, y_val = train_test_split(\n        X_images, y, test_size=config.val_size, random_state=config.seed\n    )\n    \n    print(f\"✓ Train: {len(X_train_img)} → {len(X_train_img) * config.num_augmentations} augmented\")\n    print(f\"✓ Val: {len(X_val_img)}\")\n    \n    # Create enhanced dataloaders with 7x augmentation\n    train_dataset = EnhancedBiomassDataset(X_train_img, y_train, config.img_size, True)\n    val_dataset = EnhancedBiomassDataset(X_val_img, y_val, config.img_size, False)\n    \n    train_loader = DataLoader(train_dataset, batch_size=config.batch_size, shuffle=True, num_workers=2, pin_memory=True)\n    val_loader = DataLoader(val_dataset, batch_size=config.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n    \n    # Part 1: Fine-tune best 2 CNNs on enhanced data\n    print(f\"\\n2️⃣ Fine-tuning {len(config.finetune_models)} CNNs on 7x augmented data...\")\n    finetuned_models = []\n    finetuned_scores = []\n    \n    for model_name in config.finetune_models:\n        weight_path = config.local_models[model_name]\n        model = BiomassModel(model_name, weight_path, config.img_size, freeze_backbone=False)\n        model = model.to(device)\n        \n        model, score = train_finetuned_model(model, train_loader, val_loader, model_name, config.finetune_epochs)\n        finetuned_models.append(model)\n        finetuned_scores.append(score)\n        \n        gc.collect()\n        torch.cuda.empty_cache()\n    \n    # Part 2: Extract features from ALL 4 models\n    print(f\"\\n3️⃣ Loading all 4 models for feature extraction...\")\n    feature_extractors = []\n    for model_name, weight_path in config.local_models.items():\n        if weight_path.exists():\n            model = BiomassModel(model_name, weight_path, config.img_size, freeze_backbone=True)\n            model = model.to(device)\n            feature_extractors.append(model)\n    \n    print(f\"\\n4️⃣ Extracting features with TTA...\")\n    X_train_features = extract_features_with_tta(feature_extractors, X_train_img, config.img_size)\n    X_val_features = extract_features_with_tta(feature_extractors, X_val_img, config.img_size)\n    \n    print(f\"✓ Feature shape: {X_train_features.shape}\")\n    \n    # Part 3: Train ML models with optimal ensemble\n    print(f\"\\n5️⃣ Training ML models with optimal ensemble...\")\n    lgb_models, xgb_models, scaler, lgb_score, xgb_score, optimal_r2 = train_ml_models(\n        X_train_features, y_train, X_val_features, y_val\n    )\n    \n    print(f\"\\n{'='*80}\")\n    print(\"FINAL VALIDATION RESULTS\")\n    print('='*80)\n    print(f\"Fine-tuned CNNs:\")\n    for i, name in enumerate(config.finetune_models):\n        print(f\"  {name}: {finetuned_scores[i]:.6f}\")\n    print(f\"\\nML Models:\")\n    print(f\"  LightGBM: {lgb_score:.6f}\")\n    print(f\"  XGBoost:  {xgb_score:.6f}\")\n    print(f\"\\n🎯 OPTIMAL ENSEMBLE R² (70% LGB + 30% XGB): {optimal_r2:.6f}\")\n    print('='*80)\n    \n    return {\n        'finetuned_models': finetuned_models,\n        'feature_extractors': feature_extractors,\n        'lgb_models': lgb_models,\n        'xgb_models': xgb_models,\n        'scaler': scaler,\n        'optimal_r2': optimal_r2,\n        'lgb_score': lgb_score,\n        'xgb_score': xgb_score\n    }\n\n# ============================================================================\n# GENERATE OPTIMAL SUBMISSION\n# ============================================================================\n\ndef generate_optimal_submission(trained_info):\n    print(\"\\n\" + \"=\"*80)\n    print(\"🎯 GENERATING OPTIMAL SUBMISSION (70% LGB + 30% XGB)\")\n    print(\"=\"*80)\n    \n    test_df = pd.read_csv(data_path / 'test.csv')\n    test_df['image_full_path'] = test_df['image_path'].apply(lambda x: str(data_path / x))\n    test_unique = test_df.drop_duplicates('image_path').reset_index(drop=True)\n    X_test_images = test_unique['image_full_path'].values\n    \n    print(f\"✓ Test images: {len(X_test_images)}\")\n    \n    # Extract features for ML models\n    print(\"\\n1️⃣ Extracting features for ML models...\")\n    X_test_features = extract_features_with_tta(trained_info['feature_extractors'], X_test_images, config.img_size)\n    X_test_scaled = trained_info['scaler'].transform(X_test_features)\n    \n    # Get ML predictions\n    print(\"\\n2️⃣ Getting ML predictions...\")\n    lgb_preds = np.zeros((len(X_test_images), 5))\n    xgb_preds = np.zeros((len(X_test_images), 5))\n    \n    for i in range(5):\n        lgb_preds[:, i] = trained_info['lgb_models'][i].predict(X_test_scaled)\n        xgb_preds[:, i] = trained_info['xgb_models'][i].predict(X_test_scaled)\n    \n    # Create optimal ensemble (70% LGB + 30% XGB)\n    print(\"\\n3️⃣ Creating optimal ensemble...\")\n    lightgbm_weight = 0.70\n    xgboost_weight = 0.30\n    \n    final_predictions = (lgb_preds * lightgbm_weight + \n                        xgb_preds * xgboost_weight)\n    final_predictions = np.maximum(final_predictions, 0)\n    \n    # Create submission\n    pred_with_id = pd.DataFrame({\n        'image_path': test_unique['image_path'].values,\n        'Dry_Clover_g': final_predictions[:, 0],\n        'Dry_Dead_g': final_predictions[:, 1],\n        'Dry_Green_g': final_predictions[:, 2],\n        'Dry_Total_g': final_predictions[:, 3],\n        'GDM_g': final_predictions[:, 4]\n    })\n    \n    submission_df = test_df[['sample_id', 'image_path', 'target_name']].merge(\n        pred_with_id, on='image_path', how='left'\n    )\n    \n    submission_df['target'] = submission_df.apply(\n        lambda row: row[row['target_name']], axis=1\n    )\n    \n    submission_df = submission_df[['sample_id', 'target']].sort_values('sample_id')\n    submission_df.to_csv('submission.csv', index=False)\n    \n    print(f\"\\n✅ OPTIMAL SUBMISSION SAVED!\")\n    print(f\"✅ Ensemble: 70% LightGBM + 30% XGBoost\")\n    print(f\"✅ Total predictions: {len(submission_df)}\")\n    print(\"\\n📊 First 10 rows:\")\n    print(submission_df.head(10))\n    \n    return submission_df\n\n# ============================================================================\n# EXECUTION\n# ============================================================================\n\nif __name__ == \"__main__\":\n    print(\"\"\"\n    ╔═══════════════════════════════════════════════════════════════════╗\n    ║         ENHANCED ENSEMBLE - 7x AUGMENTATION + OPTIMAL WEIGHTS     ║\n    ║                                                                   ║\n    ║  Part 1: 7x data augmentation (357 → 2499 training samples)      ║\n    ║  Part 2: Fine-tune CNNs on enhanced dataset                      ║\n    ║  Part 3: Extract features from all 4 CNNs with TTA               ║\n    ║  Part 4: Train LightGBM + XGBoost on features                    ║\n    ║  Part 5: Optimal ensemble: 70% LGB + 30% XGB                     ║\n    ║                                                                   ║\n    ║  Target: 0.68-0.72 R² (significant improvement)                  ║\n    ╚═══════════════════════════════════════════════════════════════════╝\n    \"\"\")\n    \n    try:\n        print(\"🚀 Starting enhanced training pipeline...\\n\")\n        \n        trained_info = main()\n        \n        print(\"\\n\" + \"=\"*80)\n        print(\"🏆 TRAINING COMPLETED!\")\n        print(\"=\"*80)\n        print(f\"\\n🎯 OPTIMAL VALIDATION R²: {trained_info['optimal_r2']:.6f}\")\n        \n        if trained_info['optimal_r2'] >= 0.70:\n            print(\"🌟🌟🌟 EXCELLENT! Better than expected!\")\n        elif trained_info['optimal_r2'] >= 0.68:\n            print(\"🎉🎉 GREAT! Target achieved!\")\n        elif trained_info['optimal_r2'] >= 0.65:\n            print(\"✅ GOOD! Solid improvement!\")\n        else:\n            print(\"✅ Baseline improved!\")\n        \n        submission_df = generate_optimal_submission(trained_info)\n        \n        print(\"\\n\" + \"=\"*80)\n        print(\"✅ READY FOR KAGGLE SUBMISSION!\")\n        print(\"=\"*80)\n        print(f\"\\n💡 Expected Leaderboard: {trained_info['optimal_r2'] - 0.02:.2f} - {trained_info['optimal_r2'] + 0.01:.2f}\")\n        print(f\"💡 Using: 70% LightGBM + 30% XGBoost\")\n        \n    except Exception as e:\n        print(f\"\\n❌ Error: {e}\")\n        import traceback\n        traceback.print_exc()\n    finally:\n        gc.collect()\n        torch.cuda.empty_cache()\n        print(\"\\n✓ Cleanup completed\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T09:14:14.612961Z","iopub.execute_input":"2025-11-24T09:14:14.613296Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Helllo\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-24T08:53:26.534607Z","iopub.execute_input":"2025-11-24T08:53:26.534831Z","iopub.status.idle":"2025-11-24T08:53:26.540930Z","shell.execute_reply.started":"2025-11-24T08:53:26.534812Z","shell.execute_reply":"2025-11-24T08:53:26.538357Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}