{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14018229,"sourceType":"datasetVersion","datasetId":8929818},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":3329,"modelId":986},{"sourceId":268942,"sourceType":"modelInstanceVersion","isSourceIdPinned":false,"modelInstanceId":230141,"modelId":251887}],"dockerImageVersionId":31193,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Multiview ","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport glob\nfrom pathlib import Path\nimport sys\nfrom tqdm.auto import tqdm\nimport json\nfrom copy import deepcopy\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport math\nimport random\n\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\n\nfrom PIL import Image\nimport cv2\n\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, Siglip2Model, Siglip2ImageProcessor, SiglipModel, SiglipImageProcessor\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.svm import LinearSVR\nfrom sklearn.dummy import DummyRegressor\n\nfrom sklearn.decomposition import PCA\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\n\nfrom dataclasses import dataclass\nfrom typing import Optional, Dict\n\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:13:38.869910Z","iopub.execute_input":"2025-12-12T16:13:38.870336Z","iopub.status.idle":"2025-12-12T16:14:28.861506Z","shell.execute_reply.started":"2025-12-12T16:13:38.870309Z","shell.execute_reply":"2025-12-12T16:14:28.860066Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    # pl.seed_everything(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n    print('seeding done!!!')\n\ndef flush():\n    import gc\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n\nseeding(42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:28.864107Z","iopub.execute_input":"2025-12-12T16:14:28.866275Z","iopub.status.idle":"2025-12-12T16:14:28.886960Z","shell.execute_reply.started":"2025-12-12T16:14:28.866217Z","shell.execute_reply":"2025-12-12T16:14:28.885629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"@dataclass\nclass Config:\n    # Data paths\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    TRAIN_DATA_PATH: Path = DATA_PATH/'train'\n    TEST_DATA_PATH: Path = DATA_PATH/'test'\n\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    seed = 42\n\ncfg = Config()\nseeding(cfg.seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:28.888207Z","iopub.execute_input":"2025-12-12T16:14:28.888590Z","iopub.status.idle":"2025-12-12T16:14:28.912759Z","shell.execute_reply.started":"2025-12-12T16:14:28.888550Z","shell.execute_reply":"2025-12-12T16:14:28.911239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def pivot_table(df: pd.DataFrame)->pd.DataFrame:\n\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\ntrain_df = pd.read_csv(cfg.DATA_PATH/'train.csv')\ntest_df = pd.read_csv(cfg.DATA_PATH/'test.csv')\ntrain_df = pivot_table(df=train_df)\ntest_df = pivot_table(df=test_df)\n\ntrain_df['image_path'] = train_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\ntest_df['image_path'] = test_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\ntest_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:28.916880Z","iopub.execute_input":"2025-12-12T16:14:28.917291Z","iopub.status.idle":"2025-12-12T16:14:29.040937Z","shell.execute_reply.started":"2025-12-12T16:14:28.917262Z","shell.execute_reply":"2025-12-12T16:14:29.039622Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    TARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)  # remove folder path, keep filename\n        .str.replace('.jpg', '', regex=False)  # remove extension\n        + '__' + melted['target_name']\n    )\n    \n    return melted[['sample_id', 'image_path', 'target_name', 'target']]\n\nt1 = melt_table(test_df)\nt1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:29.042112Z","iopub.execute_input":"2025-12-12T16:14:29.042411Z","iopub.status.idle":"2025-12-12T16:14:29.064967Z","shell.execute_reply.started":"2025-12-12T16:14:29.042386Z","shell.execute_reply":"2025-12-12T16:14:29.063845Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head(1)\ntrain_df['Species'].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:29.066111Z","iopub.execute_input":"2025-12-12T16:14:29.067083Z","iopub.status.idle":"2025-12-12T16:14:29.095050Z","shell.execute_reply.started":"2025-12-12T16:14:29.067052Z","shell.execute_reply":"2025-12-12T16:14:29.093719Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.cluster import KMeans\n# from sklearn.model_selection import StratifiedGroupKFold\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.cluster import KMeans\nfrom sklearn.model_selection import StratifiedGroupKFold\n\ndef create_robust_cv(train_df, \n                     target_names, \n                     weights, \n                     n_splits=5, \n                     n_clusters=50, \n                     seed=42):\n    \"\"\"\n    Creates a robust CV scheme that simulates unseen test environments.\n    \"\"\"\n    df = train_df.copy()\n\n    # ----------------------------\n    # 1. Select Embedding Columns\n    # ----------------------------\n    # Robust selector for \"emb\", \"embedding\", or numeric columns\n    emb_cols = [c for c in df.columns if \"emb\" in str(c).lower()]\n    if not emb_cols:\n        exclude = set(target_names + list(weights.keys()) + ['fold', 'id', 'image_path', 'State', 'Species'])\n        emb_cols = [c for c in df.columns if c not in exclude and str(c).isdigit()]\n    \n    print(f\"Using {len(emb_cols)} embedding columns for clustering.\")\n\n    # ----------------------------\n    # 2. Visual Clustering (Define the Groups)\n    # ----------------------------\n    # We create ~50 groups. The model must learn to generalize *across* groups.\n    X_emb = df[emb_cols].values\n    kmeans = KMeans(n_clusters=n_clusters, random_state=seed, n_init=10)\n    df[\"visual_cluster\"] = kmeans.fit_predict(X_emb)\n\n    # ----------------------------\n    # 3. Create Stratification Target\n    # ----------------------------\n    # We want folds to be balanced by BIOMASS, not by cluster ID.\n    composite = np.zeros(len(df))\n    for t in target_names:\n        composite += df[t] * weights.get(t, 0)\n        \n    # Bin the continuous target into 10 bins for stratification\n    try:\n        df['target_bins'] = pd.qcut(composite, q=10, labels=False, duplicates='drop')\n    except ValueError:\n        df['target_bins'] = pd.qcut(composite, q=5, labels=False, duplicates='drop')\n\n    # ----------------------------\n    # 4. Stratified Group K-Fold\n    # ----------------------------\n    # CRITICAL: \n    #   y      = target_bins    (Balance the biomass difficulty)\n    #   groups = visual_cluster (Prevent leakage of similar images)\n    sgkf = StratifiedGroupKFold(n_splits=n_splits, shuffle=True, random_state=seed)\n\n    df[\"fold\"] = -1\n    for fold, (train_idx, val_idx) in enumerate(sgkf.split(df, df['target_bins'], groups=df[\"visual_cluster\"])):\n        df.loc[val_idx, \"fold\"] = fold\n\n    return df\n\n# ==========================================\n# CONFIGURATION\n# ==========================================\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2, 'Dry_Total_g': 0.5,\n}\n\n# LOAD AND CREATE FOLDS\ntrain_df = pd.read_csv(\"/kaggle/input/csiro-image-embeddings/train_siglip_embeddings.csv\")\ntrain_df = create_robust_cv(train_df, TARGET_NAMES, weights, n_splits=5, n_clusters=50)\n\n# VERIFICATION\nprint(\"\\nFold Distribution (Will be uneven - this is expected):\")\nprint(train_df['fold'].value_counts())\n\nprint(\"\\nVerify Leakage (Must be 0):\")\nleakage_check = train_df.groupby('visual_cluster')['fold'].nunique()\nprint(f\"Clusters appearing in multiple folds: {sum(leakage_check > 1)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:29.096890Z","iopub.execute_input":"2025-12-12T16:14:29.098599Z","iopub.status.idle":"2025-12-12T16:14:30.045481Z","shell.execute_reply.started":"2025-12-12T16:14:29.098565Z","shell.execute_reply":"2025-12-12T16:14:30.044219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    \n    patches = []\n    coords  = []   # (y1, x1, y2, x2)\n    \n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y1 = y\n            x1 = x\n            y2 = y + patch_size\n            x2 = x + patch_size\n            \n            # Pad last patch if needed (very rare with your fixed 1000×2000)\n            patch = image[y1:y2, x1:x2, :]\n            if patch.shape[0] < patch_size or patch.shape[1] < patch_size:\n                pad_h = patch_size - patch.shape[0]\n                pad_w = patch_size - patch.shape[1]\n                patch = np.pad(patch, ((0,pad_h), (0,pad_w), (0,0)), mode='reflect')\n            \n            patches.append(patch)\n            coords.append((y1, x1, y2, x2))\n    \n    return patches, coords","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:30.046463Z","iopub.execute_input":"2025-12-12T16:14:30.046852Z","iopub.status.idle":"2025-12-12T16:14:30.056004Z","shell.execute_reply.started":"2025-12-12T16:14:30.046815Z","shell.execute_reply":"2025-12-12T16:14:30.054122Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_model(model_path: str, device: str = 'cpu'):\n    model = AutoModel.from_pretrained(\n        model_path,\n        local_files_only=True\n    )\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    return model.eval().to(device), processor\n\ndino_path = \"/kaggle/input/dinov2/pytorch/giant/1\"\nsiglip_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n\n# dino_model, dino_processor = get_model(\n#     model_path=dino_path, device=device\n# )\n\n# siglip_model, siglip_processor = get_model(\n#     model_path=siglip_path, device=device\n# )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:30.057192Z","iopub.execute_input":"2025-12-12T16:14:30.057656Z","iopub.status.idle":"2025-12-12T16:14:30.085836Z","shell.execute_reply.started":"2025-12-12T16:14:30.057630Z","shell.execute_reply":"2025-12-12T16:14:30.084019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_embeddings(model_path, df, patch_size=520):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    print(device)\n\n    model, processor = get_model(\n        model_path=model_path, device=device\n    )\n\n    IMAGE_PATHS = []\n    EMBEDDINGS = []\n\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        img_path = row['image_path']\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        patches, coords = split_image(img, patch_size=patch_size)\n        images = [Image.fromarray(p).convert(\"RGB\") for p in patches]\n\n        inputs = processor(images=images, return_tensors=\"pt\").to(model.device)\n        with torch.no_grad():\n            if 'siglip' in model_path:\n                features = model.get_image_features(**inputs)\n            elif 'dino' in model_path:\n                features = model(**inputs).pooler_output\n                # patches = model(**inputs).last_hidden_state\n                # features = patches[:, 0, :]\n            else:\n                raise Exception(\"Model should be dino or siglip\")\n        embeds = features.mean(dim=0).detach().cpu().numpy()\n        EMBEDDINGS.append(embeds)\n        IMAGE_PATHS.append(img_path)\n\n    embeddings = np.stack(EMBEDDINGS, axis=0)\n    n_features = embeddings.shape[1]\n    emb_columns = [f\"emb{i+1}\" for i in range(n_features)]\n    emb_df = pd.DataFrame(embeddings, columns=emb_columns)\n    emb_df['image_path'] = IMAGE_PATHS\n    df_final = df.merge(emb_df, on='image_path', how='left')\n    flush()\n    return df_final \n\n# train_siglip_df = compute_embeddings(model_path=siglip_path, df=train_df, patch_size=520)\ntrain_siglip_df = train_df.copy()\ntest_siglip_df = compute_embeddings(model_path=siglip_path, df=test_df, patch_size=520)\n\nflush()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:14:30.089078Z","iopub.execute_input":"2025-12-12T16:14:30.089430Z","iopub.status.idle":"2025-12-12T16:15:53.191921Z","shell.execute_reply.started":"2025-12-12T16:14:30.089404Z","shell.execute_reply":"2025-12-12T16:15:53.190614Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Text embeddings","metadata":{}},{"cell_type":"code","source":"# \"dense pasture grass\",\n#         \"sparse pasture vegetation\",\n#         \"patchy grass cover\",\n#         \"bare soil patches in grass\",\n#         \"thick tangled grass\",\n#         \"open low-density pasture\",\n#         \"dry cracked soil\",\n#         \"dry canopy\",\n#         \"low moisture vegetation\",\n#         \"dry pasture with yellow tones\",\n#         \"wilted grass\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:15:53.193096Z","iopub.execute_input":"2025-12-12T16:15:53.193373Z","iopub.status.idle":"2025-12-12T16:15:53.198741Z","shell.execute_reply.started":"2025-12-12T16:15:53.193352Z","shell.execute_reply":"2025-12-12T16:15:53.197605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport numpy as np\nimport pandas as pd\nfrom transformers import AutoModel, AutoTokenizer\n\n# Your specific path\nSIGLIP_PATH = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n\ndef generate_semantic_features(image_embeddings, model_path=SIGLIP_PATH):\n    \"\"\"\n    Projects image embeddings onto agronomic text concepts.\n    Returns: (N_samples, N_prompts) numpy array\n    \"\"\"\n    print(f\"Loading SigLIP Text Encoder from {model_path}...\")\n\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    # 1. Load Model (Text part only ideally, but SigLIP usually loads all)\n    try:\n        model = AutoModel.from_pretrained(model_path).to(device)\n        tokenizer = AutoTokenizer.from_pretrained(model_path)\n    except Exception as e:\n        print(f\"Error loading model: {e}\")\n        return None\n\n    # 2. Define Agronomic Prompts\n    # These act as \"Virtual Sensors\" for specific pasture traits\n    prompts = [\n        \"dense pasture grass\",\n        \"open low-density pasture\",\n        \"sparse pasture vegetation\",\n        \"patchy grass cover\",\n        \"bare soil patches\",\n        \"thick tangled grass\",\n        \"overgrazed pasture\",\n        \"ungrazed mature pasture\",\n        \"pasture with accumulated dead material\",\n        \"standing dead biomass\",\n        \"tall grass\",\n        \"short grass\",\n        \"overgrown grass\",\n        \"recently grazed pasture\",\n        \"clover pasture\",\n        \"white clover pasture\",\n        \"subclover pasture\",\n        \"lucerne pasture\",\n        \"ryegrass pasture\",\n        \"phalaris pasture\",\n        \"fescue pasture\",\n        \"mixed pasture grasses\",\n        \"ryegrass and clover mix\",\n        \"phalaris and clover pasture\",\n        \"ryegrass clover phalaris mix\"\n    ]\n    \n    # 3. Encode Text\n    print(\"Encoding prompts...\")\n    text_inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(device)\n    with torch.no_grad():\n        text_emb = model.get_text_features(**text_inputs)\n        # Normalize text embeddings\n        text_emb = text_emb / text_emb.norm(p=2, dim=-1, keepdim=True)\n    \n    # 4. Compute Similarity (Dot Product)\n    # Ensure image_embeddings is a torch tensor on the right device\n    if isinstance(image_embeddings, np.ndarray):\n        img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n    else:\n        img_tensor = image_embeddings.to(device)\n        \n    # Normalize image embeddings\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n    \n    # Matrix Multiplication: (N_imgs, 768) @ (768, N_prompts) -> (N_imgs, N_prompts)\n    semantic_scores = torch.matmul(img_tensor, text_emb.T).cpu().numpy()\n    \n    # Clean up\n    del model, text_inputs, text_emb, img_tensor\n    torch.cuda.empty_cache()\n    \n    return semantic_scores\n\n# --- USAGE ---\n# Assuming 'embeddings_all' is your full (N, 768) array of SigLIP image embeddings\n# If you have train/test split, concat them first to generate features for all\n# semantic_feats_all = generate_semantic_features(embeddings_all)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:03.816087Z","iopub.execute_input":"2025-12-12T16:41:03.816422Z","iopub.status.idle":"2025-12-12T16:41:03.829013Z","shell.execute_reply.started":"2025-12-12T16:41:03.816390Z","shell.execute_reply":"2025-12-12T16:41:03.827807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature engineering","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.preprocessing import StandardScaler, PolynomialFeatures\nfrom sklearn.metrics.pairwise import cosine_similarity\n\nclass SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n    def __init__(self, \n                 n_pca=0.95, \n                 n_pls=8,\n                 n_gmm=5,\n                 random_state=42):\n        \n        self.n_pca = n_pca\n        self.n_pls = n_pls\n        self.n_gmm = n_gmm\n        self.random_state = random_state\n        \n        # Transformers\n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n\n        # Internal State\n        self.global_mean_ = None\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        \"\"\"\n        X: Raw Embeddings (N, D)\n        y: Targets\n        X_semantic: (Optional) Semantic Scores (N, K)\n        \"\"\"\n        \n        # 1. Scale Raw Embeddings\n        X_scaled = self.scaler.fit_transform(X)\n        self.global_mean_ = np.mean(X_scaled, axis=0)\n        \n        # 2. Fit Unsupervised transformers\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        \n        # 3. Fit PLS if y is provided\n        if y is not None:\n            y_clean = y.values if hasattr(y, 'values') else y\n            if y_clean.ndim == 1:\n                y_clean = y_clean.reshape(-1, 1)\n\n            self.pls.fit(X_scaled, y_clean)\n            self.pls_fitted_ = True\n        \n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        return self._generate_features(X_scaled, X_semantic)\n\n    def _generate_features(self, X_scaled, X_semantic=None):\n        features = []\n        \n        # A. Optional Semantic Features\n        if X_semantic is not None:\n            features.append(X_semantic)\n            if X_semantic.shape[1] >= 2:\n                interact = (X_semantic[:, 0] * X_semantic[:, 2]).reshape(-1, 1)\n                features.append(interact)\n\n        # B. PCA components\n        f_pca = self.pca.transform(X_scaled)\n        features.append(f_pca)\n        \n        # C. PLS components\n        if self.pls_fitted_:\n            f_pls = self.pls.transform(X_scaled)\n            features.append(f_pls)\n        \n        # D. GMM cluster probabilities\n        f_gmm = self.gmm.predict_proba(X_scaled)\n        features.append(f_gmm)\n        \n        # E. Norm + Typicality\n        f_mag = np.linalg.norm(X_scaled, axis=1, keepdims=True)\n        f_sim = cosine_similarity(X_scaled, self.global_mean_.reshape(1, -1))\n        features.extend([f_mag, f_sim])\n        \n        return np.hstack(features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:04.187218Z","iopub.execute_input":"2025-12-12T16:41:04.187511Z","iopub.status.idle":"2025-12-12T16:41:04.201587Z","shell.execute_reply.started":"2025-12-12T16:41:04.187490Z","shell.execute_reply":"2025-12-12T16:41:04.200123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"COLUMNS = train_df.filter(like=\"emb\").columns\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\n\nTARGET_MAX = {\n    \"Dry_Clover_g\": 71.7865,\n    \"Dry_Dead_g\": 83.8407,\n    \"Dry_Green_g\": 157.9836,\n    \"Dry_Total_g\": 185.70,\n    \"GDM_g\": 157.9836,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(TARGET_NAMES):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(TARGET_NAMES):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n\n    return 1 - ss_res / ss_tot","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:04.321967Z","iopub.execute_input":"2025-12-12T16:41:04.322274Z","iopub.status.idle":"2025-12-12T16:41:04.332584Z","shell.execute_reply.started":"2025-12-12T16:41:04.322250Z","shell.execute_reply":"2025-12-12T16:41:04.331411Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# def post_process_biomass(df_preds):\n#     \"\"\"\n#     Enforces physical mass balance constraints on biomass predictions.\n    \n#     Constraints enforced:\n#     1. Dry_Green_g + Dry_Clover_g = GDM_g\n#     2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \n#     Method:\n#     Uses Orthogonal Projection. It finds the set of values that satisfy\n#     the constraints while minimizing the Euclidean distance to the \n#     original model predictions.\n    \n#     Args:\n#         df_preds (pd.DataFrame): DataFrame containing the 5 prediction columns.\n        \n#     Returns:\n#         pd.DataFrame: A new DataFrame with consistent, non-negative values.\n#     \"\"\"\n#     # 1. Define the specific order required for the math\n#     # We treat the vector x as: [Green, Clover, Dead, GDM, Total]\n#     ordered_cols = [\n#         \"Dry_Green_g\", \n#         \"Dry_Clover_g\", \n#         \"Dry_Dead_g\", \n#         \"GDM_g\", \n#         \"Dry_Total_g\"\n#     ]\n    \n#     # Check if columns exist\n#     if not all(col in df_preds.columns for col in ordered_cols):\n#         missing = [c for c in ordered_cols if c not in df_preds.columns]\n#         raise ValueError(f\"Input DataFrame is missing columns: {missing}\")\n\n#     # 2. Extract values in the specific order -> Shape (N_samples, 5)\n#     Y = df_preds[ordered_cols].values.T  # Transpose to (5, N) for matrix math\n\n#     # 3. Define the Constraint Matrix C\n#     # We want Cx = 0\n#     # Eq 1: 1*Green + 1*Clover + 0*Dead - 1*GDM + 0*Total = 0\n#     # Eq 2: 0*Green + 0*Clover + 1*Dead + 1*GDM - 1*Total = 0\n#     C = np.array([\n#         [1, 1, 0, -1,  0],\n#         [0, 0, 1,  1, -1]\n#     ])\n\n#     # 4. Calculate Projection Matrix P\n#     # P = I - C^T * (C * C^T)^-1 * C\n#     # This projects any vector onto the null space of C (the valid subspace)\n#     C_T = C.T\n#     inv_CCt = np.linalg.inv(C @ C_T)\n#     P = np.eye(5) - C_T @ inv_CCt @ C\n\n#     # 5. Apply Projection\n#     # Y_new = P * Y\n#     Y_reconciled = P @ Y\n\n#     # 6. Transpose back to (N, 5)\n#     Y_reconciled = Y_reconciled.T\n\n#     # 7. Post-correction for negatives\n#     # Projection can mathematically create negative values (e.g. if Total was predicted 0)\n#     # We clip to 0. Note: This might slightly break the sum equality again, \n#     # but exact equality with negatives is physically impossible anyway.\n#     Y_reconciled = Y_reconciled.clip(min=0)\n\n#     # 8. Create Output DataFrame\n#     df_out = df_preds.copy()\n#     df_out[ordered_cols] = Y_reconciled\n\n#     return df_out\n\ndef post_process_biomass(df_preds):\n    \"\"\"\n    Enforces mass balance constraints hierarchically.\n    \n    Philosophy: \n    - GDM_g is trusted. Green/Clover are scaled to match it.\n    - Dry_Total_g is trusted. Dead is derived as (Total - GDM).\n    - If constraints are physically impossible (e.g. GDM > Total),\n      we assume Total was underestimated and raise it to match GDM.\n      \n    Args:\n        df_preds (pd.DataFrame): Predictions\n        \n    Returns:\n        pd.DataFrame: Consistently processed dataframe.\n    \"\"\"\n    # Create a copy to avoid SettingWithCopy warnings\n    df_out = df_preds.copy()\n    \n    # ---------------------------------------------------------\n    # 1. Enforce: Dry_Green_g + Dry_Clover_g = GDM_g\n    # ---------------------------------------------------------\n    # We trust the *magnitude* of GDM_g more than the components.\n    # We trust the *ratio* of Green vs Clover from the model.\n    \n    # Calculate current component sum\n    comp_sum = df_out[\"Dry_Green_g\"] + df_out[\"Dry_Clover_g\"]\n    \n    # Avoid division by zero\n    mask_nonzero = comp_sum > 1e-9\n    \n    # Calculate scaling factor so components sum exactly to GDM\n    scale_factor = df_out.loc[mask_nonzero, \"GDM_g\"] / comp_sum[mask_nonzero]\n    \n    # Apply scaling\n    df_out.loc[mask_nonzero, \"Dry_Green_g\"] *= scale_factor\n    df_out.loc[mask_nonzero, \"Dry_Clover_g\"] *= scale_factor\n    \n    # Edge case: If comp_sum is 0 but GDM is not, we can't scale.\n    # (Optional: You could split GDM evenly, but usually the model predicts 0 GDM here too)\n    \n    # ---------------------------------------------------------\n    # 2. Enforce: GDM_g + Dry_Dead_g = Dry_Total_g\n    # ---------------------------------------------------------\n    # You stated Dead is hard to predict. \n    # Therefore, we discard the direct prediction of Dead and derive it.\n    \n    df_out[\"Dry_Dead_g\"] = df_out[\"Dry_Total_g\"] - df_out[\"GDM_g\"]\n    \n    # ---------------------------------------------------------\n    # 3. Handle Physical Impossibilities (Negative Dead)\n    # ---------------------------------------------------------\n    # If Dead < 0, it means GDM > Total. This is physically impossible.\n    # Logic: GDM is a sum of living parts (robust). Total is the scan of everything.\n    # If GDM > Total, the model likely underestimated Total.\n    \n    neg_dead_mask = df_out[\"Dry_Dead_g\"] < 0\n    \n    if neg_dead_mask.any():\n        # Set Dead to 0 (cannot be negative)\n        df_out.loc[neg_dead_mask, \"Dry_Dead_g\"] = 0\n        \n        # Raise Total to match GDM (maintaining balance)\n        df_out.loc[neg_dead_mask, \"Dry_Total_g\"] = df_out.loc[neg_dead_mask, \"GDM_g\"]\n\n    return df_out\n\ndef compare_results(oof, train_data):\n    y_oof_df = pd.DataFrame(oof, columns=TARGET_NAMES) # ensure columns match\n    # 2. Check Score BEFORE Processing\n    raw_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_df.values)\n    print(f\"Raw CV Score: {raw_score:.6f}\")\n    \n    # 3. Apply Post-Processing\n    y_oof_proc = post_process_biomass(y_oof_df)\n    \n    # 4. Check Score AFTER Processing\n    proc_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_proc.values)\n    print(f\"Processed CV Score: {proc_score:.6f}\")\n    \n    print(f\"Improvement: {raw_score - proc_score:.6f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:04.500319Z","iopub.execute_input":"2025-12-12T16:41:04.500617Z","iopub.status.idle":"2025-12-12T16:41:04.512167Z","shell.execute_reply.started":"2025-12-12T16:41:04.500596Z","shell.execute_reply":"2025-12-12T16:41:04.510902Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# from sklearn.preprocessing import PowerTransformer, QuantileTransformer, RobustScaler\n# from sklearn.svm import SVR\n\n# def cross_validate(model, train_data, test_data, feature_engine, target_transform='max', n_splits=5, seed=42):\n#     \"\"\"\n#     target_transform options:\n#     - 'max': Linear scaling by max value (Preserves distribution shape)\n#     - 'log': np.log1p (Aggressive compression of outliers)\n#     - 'sqrt': np.sqrt (Moderate compression, good for biological counts/area)\n#     - 'yeo-johnson': PowerTransformer (Makes data Gaussian-like automatically)\n#     - 'quantile': QuantileTransformer (Forces strict Normal distribution)\n#     \"\"\"\n    \n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES]\n    \n#     y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n#     y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n#     for fold in range(n_splits):\n#         seeding(seed*(seed//2 + fold))\n#         # 1. Split Data\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         X_train_raw = train_data[train_mask][COLUMNS].values\n#         X_valid_raw = train_data[valid_mask][COLUMNS].values\n#         X_test_raw = test_data[COLUMNS].values\n        \n#         y_train = train_data[train_mask][TARGET_NAMES].values\n#         y_valid = train_data[valid_mask][TARGET_NAMES].values\n\n#         # ===========================\n#         # 2) TARGET TRANSFORMATION\n#         # ===========================\n#         transformer = None # To store stateful transformers (Yeo/Quantile)\n        \n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n            \n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n            \n#         elif target_transform == 'sqrt':\n#             # Great for biomass/area data (Variance stabilizing)\n#             y_train_proc = np.sqrt(y_train)\n            \n#         elif target_transform == 'yeo-johnson':\n#             # Learns optimal parameter to make data Gaussian\n#             transformer = PowerTransformer(method='yeo-johnson', standardize=True)\n#             y_train_proc = transformer.fit_transform(y_train)\n            \n#         elif target_transform == 'quantile':\n#             # Forces data into a normal distribution (Robust to outliers)\n#             # transformer = QuantileTransformer(output_distribution='uniform', n_quantiles=64, random_state=42)\n#             transformer = RobustScaler()\n#             y_train_proc = transformer.fit_transform(y_train)\n            \n#         else:\n#             y_train_proc = y_train\n\n#         # ==========================================\n#         # 3) FEATURE ENGINEERING\n#         # ==========================================\n#         engine = deepcopy(feature_engine)\n#         # Note: If your engine uses PLS, pass the transformed y!\n#         engine.fit(X_train_raw, y=y_train_proc) \n        \n#         x_train_eng = engine.transform(X_train_raw)\n#         x_valid_eng = engine.transform(X_valid_raw)\n#         x_test_eng = engine.transform(X_test_raw)\n        \n#         # ==========================================\n#         # 4) TRAIN & PREDICT\n#         # ==========================================\n#         fold_valid_pred = np.zeros_like(y_valid)\n#         fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n\n#         for k in range(len(TARGET_NAMES)):\n#             regr = deepcopy(model)\n#             regr.fit(x_train_eng, y_train_proc[:, k])\n            \n#             # Raw Predictions (in transformed space)\n#             pred_valid_raw = regr.predict(x_valid_eng)\n#             pred_test_raw = regr.predict(x_test_eng)\n            \n#             # Store raw for inverse transform block below\n#             fold_valid_pred[:, k] = pred_valid_raw\n#             fold_test_pred[:, k] = pred_test_raw\n\n#         # ===========================\n#         # 5) INVERSE TRANSFORM (Apply to full matrix)\n#         # ===========================\n#         if target_transform == 'log':\n#             fold_valid_pred = np.expm1(fold_valid_pred)\n#             fold_test_pred = np.expm1(fold_test_pred)\n            \n#         elif target_transform == 'max':\n#             fold_valid_pred = fold_valid_pred * target_max_arr\n#             fold_test_pred = fold_test_pred * target_max_arr\n            \n#         elif target_transform == 'sqrt':\n#             # Inverse of sqrt is square\n#             fold_valid_pred = np.square(fold_valid_pred)\n#             fold_test_pred = np.square(fold_test_pred)\n            \n#         elif target_transform in ['yeo-johnson', 'quantile']:\n#             # Use the fitted transformer to invert\n#             fold_valid_pred = transformer.inverse_transform(fold_valid_pred)\n#             fold_test_pred = transformer.inverse_transform(fold_test_pred)\n\n#         # # Final Clip (Biomass cannot be negative)\n#         # fold_valid_pred = fold_valid_pred.clip(min=0)\n#         # fold_test_pred = fold_test_pred.clip(min=0)\n\n#         # Store results\n#         y_pred.loc[val_idx] = fold_valid_pred\n#         y_pred_test += fold_test_pred / n_splits\n        \n#         if fold == 0:\n#             print(f\"  [Fold 0] Target: {target_transform}, Feats: {x_train_eng.shape}\")\n\n#     full_cv = competition_metric(y_true.values, y_pred.values)\n#     print(f\"Full CV Score: {full_cv:.6f}\")\n    \n#     return y_pred.values, y_pred_test\n\n# # Initialize\n# feat_engine = SupervisedEmbeddingEngine(\n#     n_pca=0.96,\n#     n_pls=8,             # Extract 8 strong supervised signals\n#     n_gmm=6,             # 6 Soft clusters\n# )\n\n# print(\"######## Ridge Regression #######\")\n# oof_ridge, pred_test_ri = cross_validate(Ridge(), train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# compare_results(oof_ridge, train_siglip_df)\n\n# print(\"####### Lasso Regression #######\")\n# oof_la, pred_test_la = cross_validate(Lasso(), train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# compare_results(oof_la, train_siglip_df)\n\n# print(\"\\n###### GradientBoosting Regressor #######\")\n# oof_gb, pred_test_gb = cross_validate(GradientBoostingRegressor(), train_siglip_df, test_siglip_df, feature_engine=feat_engine, target_transform='max')\n# compare_results(oof_gb, train_siglip_df)\n\n# print(\"\\n###### Hist Gradient Boosting Regressor ######\")\n# oof_hb, pred_test_hb = cross_validate(HistGradientBoostingRegressor(), train_siglip_df, test_siglip_df, feature_engine=feat_engine, target_transform='max')\n# compare_results(oof_hb, train_siglip_df)\n\n# print(\"\\n##### CAT Regressor ######\")\n# oof_cat, pred_test_cat = cross_validate(\n#     CatBoostRegressor(verbose=0), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine\n# )\n# compare_results(oof_cat, train_siglip_df)\n\n# print(\"\\n######## XGB #######\")\n# oof_xgb, pred_test_xgb = cross_validate(\n#     XGBRegressor(verbosity=0), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_xgb, train_siglip_df)\n\n# print(\"\\n######## LGBM #######\")\n# oof_lgbm, pred_test_lgbm = cross_validate(\n#     LGBMRegressor(verbose=-1), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     target_transform='max')\n# compare_results(oof_lgbm, train_siglip_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:10.203793Z","iopub.execute_input":"2025-12-12T16:41:10.204076Z","iopub.status.idle":"2025-12-12T16:41:10.214609Z","shell.execute_reply.started":"2025-12-12T16:41:10.204057Z","shell.execute_reply":"2025-12-12T16:41:10.212726Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Semantic probing","metadata":{}},{"cell_type":"code","source":"# --- STEP 1: Generate Semantic Features ---\n# We combine Train and Test to generate features in one go, then split them back.\n# This ensures the text-projections are consistent.\n\n# Concatenate embeddings\nX_all_emb = np.vstack([\n    train_siglip_df[COLUMNS].values, \n    test_siglip_df[COLUMNS].values\n])\n\n# Generate Semantic Probes (Using the function defined in Part 1)\n# Make sure SIGLIP_PATH is correct for your environment\nprint(\"Generating Semantic Features via SigLIP Text Encoder...\")\ntry:\n    all_semantic_scores = generate_semantic_features(X_all_emb, model_path=SIGLIP_PATH)\n    \n    # Split back into Train and Test\n    n_train = len(train_siglip_df)\n    sem_train_full = all_semantic_scores[:n_train]\n    sem_test_full = all_semantic_scores[n_train:]\n    print(f\"Semantic Features Generated. Train: {sem_train_full.shape}, Test: {sem_test_full.shape}\")\n    \nexcept Exception as e:\n    print(f\"Skipping Semantic Features due to error: {e}\")\n    # Fallback to None if model path is wrong or memory fails\n    sem_train_full = None\n    sem_test_full = None","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:10.597152Z","iopub.execute_input":"2025-12-12T16:41:10.597576Z","iopub.status.idle":"2025-12-12T16:41:23.771059Z","shell.execute_reply.started":"2025-12-12T16:41:10.597551Z","shell.execute_reply":"2025-12-12T16:41:23.769653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# --- STEP 2: Updated Cross-Validation Function ---\ndef cross_validate(model, train_data, test_data, feature_engine, \n                   semantic_train=None, semantic_test=None, # <--- NEW ARGS\n                   target_transform='max', n_splits=5, seed=42):\n    \n    # Setup Targets\n    target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n    y_true = train_data[TARGET_NAMES]\n    \n    # Setup Storage\n    y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n    y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n    for fold in range(n_splits):\n        seeding(seed*(seed//2 + fold))\n        # Create masks\n        train_mask = train_data['fold'] != fold\n        valid_mask = train_data['fold'] == fold\n        val_idx = train_data[valid_mask].index\n\n        # Raw Inputs (Embeddings)\n        X_train_raw = train_data[train_mask][COLUMNS].values\n        X_valid_raw = train_data[valid_mask][COLUMNS].values\n        X_test_raw = test_data[COLUMNS].values\n        \n        # Semantic Inputs (Slicing)\n        # We handle the case where semantic features might be None\n        sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n        sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        \n        # Raw Targets\n        y_train = train_data[train_mask][TARGET_NAMES].values\n        y_valid = train_data[valid_mask][TARGET_NAMES].values\n\n        # ===========================\n        # 1) TRANSFORM TARGETS\n        # ===========================\n        if target_transform == 'log':\n            y_train_proc = np.log1p(y_train)\n        elif target_transform == 'max':\n            y_train_proc = y_train / target_max_arr\n        else:\n            y_train_proc = y_train\n\n        # ==========================================\n        # 2) FEATURE ENGINEERING\n        # ==========================================\n        engine = deepcopy(feature_engine)\n        \n        # FIT: Now passes y (for PLS/RFE) and Semantic Features\n        engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        \n        # TRANSFORM: Pass Semantic Features\n        x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n        x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n        # For test, we use the full test semantic set\n        x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n        \n        # ==========================================\n        # 3) TRAIN & PREDICT\n        # ==========================================\n        fold_valid_pred = np.zeros_like(y_valid)\n        fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n\n        for k in range(len(TARGET_NAMES)):\n            regr = deepcopy(model)\n            \n            # Fit model\n            regr.fit(x_train_eng, y_train_proc[:, k])\n            \n            # Predict\n            pred_valid_raw = regr.predict(x_valid_eng)\n            pred_test_raw = regr.predict(x_test_eng)\n            \n            # ===========================\n            # 4) INVERSE TRANSFORM\n            # ===========================\n            if target_transform == 'log':\n                pred_valid_inv = np.expm1(pred_valid_raw)\n                pred_test_inv = np.expm1(pred_test_raw)\n            elif target_transform == 'max':\n                pred_valid_inv = (pred_valid_raw * target_max_arr[k])\n                pred_test_inv = (pred_test_raw * target_max_arr[k])\n            else:\n                pred_valid_inv = pred_valid_raw\n                pred_test_inv = pred_test_raw\n\n            fold_valid_pred[:, k] = pred_valid_inv\n            fold_test_pred[:, k] = pred_test_inv\n\n        # Store results\n        y_pred.loc[val_idx] = fold_valid_pred\n        y_pred_test += fold_test_pred / n_splits\n        \n        if fold == 0:\n            print(f\"  [Fold 0 Info] Target: {target_transform}, Feats: {x_train_eng.shape}\")\n\n    full_cv = competition_metric(y_true.values, y_pred.values)\n    print(f\"Full CV Score: {full_cv:.6f}\")\n    \n    return y_pred.values, y_pred_test\n\n# --- STEP 3: Run Models ---\n\n# Initialize the NEW Supervised Engine\nfeat_engine = SupervisedEmbeddingEngine(\n    n_pca=0.80,\n    n_pls=8,             # Supervised signals\n    n_gmm=6,             # Soft clusters\n)\n\n# print(\"######## Ridge Regression #######\")\n# oof_ridge, pred_test_ri = cross_validate(\n#     Ridge(), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full, # <--- Pass Semantic\n#     semantic_test=sem_test_full    # <--- Pass Semantic\n# )\n# compare_results(oof_ridge, train_siglip_df)\n\n# print(\"\\n####### Lasso Regression #######\")\n# # Lasso should perform much better now due to PLS and RFE\n# oof_la, pred_test_la = cross_validate(\n#     Lasso(alpha=0.015), # Small alpha for normalized feats\n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full\n# )\n# compare_results(oof_la, train_siglip_df)\n\nprint(\"\\n###### GradientBoosting Regressor #######\")\noof_gb, pred_test_gb = cross_validate(\n    GradientBoostingRegressor(), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_gb, train_siglip_df)\n\nprint(\"\\n###### Hist Gradient Boosting Regressor ######\")\noof_hb, pred_test_hb = cross_validate(\n    HistGradientBoostingRegressor(), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_hb, train_siglip_df)\n\nprint(\"\\n##### CAT Regressor ######\")\noof_cat, pred_test_cat = cross_validate(\n    CatBoostRegressor(verbose=0), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_cat, train_siglip_df)\n\n# print(\"\\n######## XGB #######\")\n# oof_xgb, pred_test_xgb = cross_validate(\n#     XGBRegressor(verbosity=0), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine, \n#     semantic_train=sem_train_full,\n#     semantic_test=sem_test_full,\n#     target_transform='max')\n# compare_results(oof_xgb, train_siglip_df)\n\nprint(\"\\n######## LGBM #######\")\noof_lgbm, pred_test_lgbm = cross_validate(\n    LGBMRegressor(verbose=-1), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine, \n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full,\n    target_transform='max')\ncompare_results(oof_lgbm, train_siglip_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:41:23.774119Z","iopub.execute_input":"2025-12-12T16:41:23.774387Z","iopub.status.idle":"2025-12-12T16:45:15.205800Z","shell.execute_reply.started":"2025-12-12T16:41:23.774368Z","shell.execute_reply":"2025-12-12T16:45:15.204591Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Hyperparameter tuning","metadata":{}},{"cell_type":"code","source":"# import optuna\n# import numpy as np\n# import pandas as pd\n# from sklearn.base import clone\n# from sklearn.decomposition import PCA\n# from sklearn.multioutput import MultiOutputRegressor\n# from xgboost import XGBRegressor\n# from lightgbm import LGBMRegressor\n# from catboost import CatBoostRegressor\n\n# # ---------------------------------------------------------\n# # 1. Corrected Helper Function\n# # ---------------------------------------------------------\n# feat_engine = EmbeddingFeatureEngine(\n#     n_pca_components=0.90, \n#     n_clusters=25, \n#     use_stats=True, \n#     use_similarity=True,\n#     use_anomaly=True,        # Adds Anomaly Score\n#     use_entropy=True,        # Adds Entropy\n#     use_pca_interactions=True # Adds Poly features on Top 5 PCA\n# )\n\n# def get_cv_score(model, train_data, feature_engine, target_transform='max', random_state=42):\n#     \"\"\"\n#     Runs CV on ALL folds dynamically to return a single score.\n#     Optimized for speed (vectorized target processing).\n    \n#     Args:\n#         model: Estimator (must support Multi-Output or be wrapped in MultiOutputRegressor)\n#         train_data: DataFrame containing 'fold' column\n#         feature_engine: Transformer with .fit() and .transform()\n#         target_transform: 'log', 'max', or None\n#     \"\"\"\n#     # 1. Setup global constants\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES].values\n#     y_pred = np.zeros([len(train_data), len(TARGET_NAMES)], dtype=float)\n    \n#     # 2. Detect Folds dynamically\n#     folds = sorted(train_data['fold'].unique())\n    \n#     # 3. Loop over folds\n#     for fold in folds:\n#         # -------------------------\n#         # Data Slicing\n#         # -------------------------\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         X_train_raw = train_data.loc[train_mask, COLUMNS].values\n#         X_valid_raw = train_data.loc[valid_mask, COLUMNS].values\n        \n#         y_train = train_data.loc[train_mask, TARGET_NAMES].values\n#         # y_valid used implicitely via y_true at the end\n\n#         # -------------------------\n#         # A) Transform Targets\n#         # -------------------------\n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n#         else:\n#             y_train_proc = y_train\n\n#         # -------------------------\n#         # B) Feature Engineering\n#         # -------------------------\n#         # Fit engine only on training split\n#         engine = deepcopy(feature_engine)\n#         engine.fit(X_train_raw)\n        \n#         X_train_eng = engine.transform(X_train_raw)\n#         X_valid_eng = engine.transform(X_valid_raw)\n\n#         # -------------------------\n#         # C) Fit Model (Multi-Output)\n#         # -------------------------\n#         regr = clone(model)\n#         regr.fit(X_train_eng, y_train_proc)\n\n#         # -------------------------\n#         # D) Predict & Inverse Transform\n#         # -------------------------\n#         valid_pred_raw = np.array(regr.predict(X_valid_eng))\n        \n#         if target_transform == 'log':\n#             valid_pred = np.expm1(valid_pred_raw)\n#         elif target_transform == 'max':\n#             valid_pred = valid_pred_raw * target_max_arr\n#         else:\n#             valid_pred = valid_pred_raw\n\n#         # Clip and Store\n#         y_pred[val_idx] = valid_pred.clip(0)\n\n#     # 4. Calculate Metric\n#     score = competition_metric(y_true, y_pred)\n    \n#     # # Clean output buffer if running in a loop\n#     # try:\n#     #     from IPython.display import flush_ipython\n#     #     flush_ipython()\n#     # except ImportError:\n#     #     pass\n        \n#     return score\n\n# # ---------------------------------------------------------\n# # 2. Corrected CatBoost Objective\n# # ---------------------------------------------------------\n# def objective_catboost(trial):\n#     params = {\n#         # Search Space\n#         'iterations': trial.suggest_int('iterations', 800, 2000), # Increased min iterations\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'depth': trial.suggest_int('depth', 4, 8), # Reduced max depth to save memory\n#         'l2_leaf_reg': trial.suggest_float('l2_leaf_reg', 1e-3, 10.0, log=True),\n#         'random_strength': trial.suggest_float('random_strength', 1e-3, 5.0, log=True),\n#         'bagging_temperature': trial.suggest_float('bagging_temperature', 0.0, 1.0),\n        \n#         # Fixed GPU Params\n#         'loss_function': 'MultiRMSE',\n#         'task_type': 'GPU',\n#         'boosting_type': 'Plain', \n#         'devices': '0',\n#         'verbose': 0,\n#         'random_state': 42,\n#         'allow_writing_files': False # Prevents creating log files\n#     }\n    \n#     model = CatBoostRegressor(**params)\n    \n#     # Removed n_splits argument; it now uses whatever is in 'train'\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)\n\n# # ---------------------------------------------------------\n# # 3. Corrected XGBoost Objective\n# # ---------------------------------------------------------\n# def objective_xgboost(trial):\n#     params = {\n#         'n_estimators': trial.suggest_int('n_estimators', 800, 2000),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'max_depth': trial.suggest_int('max_depth', 3, 8),\n#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0.1, 10.0, log=True),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.1, 10.0, log=True),\n        \n#         # Fixed\n#         'tree_method': 'hist',\n#         'device': 'cuda',\n#         'n_jobs': -1,\n#         'random_state': 42,\n#         'verbosity': 0\n#     }\n    \n#     model = MultiOutputRegressor(XGBRegressor(**params))\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)\n\n# # ---------------------------------------------------------\n# # 4. Corrected LightGBM Objective\n# # ---------------------------------------------------------\n# def objective_lgbm(trial):\n#     params = {\n#         'n_estimators': trial.suggest_int('n_estimators', 800, 2000),\n#         'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.1, log=True),\n#         'num_leaves': trial.suggest_int('num_leaves', 20, 100),\n#         'min_child_samples': trial.suggest_int('min_child_samples', 10, 50),\n#         'subsample': trial.suggest_float('subsample', 0.6, 1.0),\n#         'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),\n#         'reg_alpha': trial.suggest_float('reg_alpha', 0.1, 10.0, log=True),\n#         'reg_lambda': trial.suggest_float('reg_lambda', 0.1, 10.0, log=True),\n        \n#         # Fixed\n#         'device': 'gpu',\n#         'n_jobs': -1,\n#         'random_state': 42,\n#         'verbose': -1\n#     }\n    \n#     model = MultiOutputRegressor(LGBMRegressor(**params))\n#     return get_cv_score(model, train_siglip_df, feature_engine=feat_engine)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.453542Z","iopub.status.idle":"2025-12-12T16:17:29.453967Z","shell.execute_reply.started":"2025-12-12T16:17:29.453748Z","shell.execute_reply":"2025-12-12T16:17:29.453765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # # --- 1. Tune CatBoost (Highest Priority) ---\n# # print(\"Tuning CatBoost...\")\n# # study_cat = optuna.create_study(direction='maximize')\n# # study_cat.optimize(objective_catboost, n_trials=20)\n\n# # print(\"Best CatBoost Params:\", study_cat.best_params)\n# # best_cat_params = study_cat.best_params\n# # # Re-add fixed params that Optuna didn't tune\n# # best_cat_params.update({\n# #     'loss_function': 'MultiRMSE', \n# #     # 'task_type': 'GPU', \n# #     'boosting_type': 'Plain', \n# #     # 'devices': '0', \n# #     'verbose': 0, \n# #     'random_state': 42\n# # })\n\n\n# # # --- 2. Tune XGBoost ---\n# # print(\"\\nTuning XGBoost...\")\n# # study_xgb = optuna.create_study(direction='maximize')\n# # study_xgb.optimize(objective_xgboost, n_trials=20)\n\n# # print(\"Best XGBoost Params:\", study_xgb.best_params)\n# # # best_xgb_params = study_xgb.best_params\n# # # best_xgb_params.update({\n# # #     'tree_method': 'hist', \n# # #     'device': 'cuda', \n# # #     'n_jobs': -1, \n# # #     'random_state': 42\n# # # })\n\n\n# # --- 3. Tune LightGBM ---\n# print(\"\\nTuning LightGBM...\")\n# study_lgbm = optuna.create_study(direction='maximize')\n# study_lgbm.optimize(objective_lgbm, n_trials=20)\n\n# print(\"Best LightGBM Params:\", study_lgbm.best_params)\n# # best_lgbm_params = study_lgbm.best_params\n# # best_lgbm_params.update({\n# #     'device': 'gpu', \n# #     'n_jobs': -1, \n# #     'random_state': 42, \n# #     'verbose': -1\n# # })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.455860Z","iopub.status.idle":"2025-12-12T16:17:29.456345Z","shell.execute_reply.started":"2025-12-12T16:17:29.456117Z","shell.execute_reply":"2025-12-12T16:17:29.456137Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Parameters\n\n### Siglip (PCA n_components=0.8)\n\n```\nBest CatBoost Params: {'iterations': 1900, 'learning_rate': 0.04488950669764926, 'depth': 4, 'l2_leaf_reg': 0.5647720146150716, 'random_strength': 0.04455012279134044, 'bagging_temperature': 0.9810426313146956}\n\nBest XGBoost Params: {'n_estimators': 1354, 'learning_rate': 0.010266591943008255, 'max_depth': 3, 'subsample': 0.6035540714532827, 'colsample_bytree': 0.9029950550994382, 'reg_alpha': 0.11110779086383878, 'reg_lambda': 9.314996597001533}\n\nBest LightGBM Params: {'n_estimators': 807, 'learning_rate': 0.014069585873331451, 'num_leaves': 48, 'min_child_samples': 19, 'subsample': 0.7451600778259232, 'colsample_bytree': 0.7457374193240649, 'reg_alpha': 0.21580623254415052, 'reg_lambda': 3.784221570035411}\n```\n\n### Siglip (NO PCA)\n\n```\nBest CatBoost Params: {'iterations': 1945, 'learning_rate': 0.025612792183534742, 'depth': 5, 'l2_leaf_reg': 0.0011451976652037553, 'random_strength': 0.03363171953662423, 'bagging_temperature': 0.9926373709983951}\n\nBest XGBoost Params: {'n_estimators': 1695, 'learning_rate': 0.013048089867977527, 'max_depth': 4, 'subsample': 0.7151550925326732, 'colsample_bytree': 0.7883122143141527, 'reg_alpha': 0.6732457617935534, 'reg_lambda': 8.692842925053135}\n\nBest LightGBM Params: {'n_estimators': 1983, 'learning_rate': 0.03365765614894731, 'num_leaves': 21, 'min_child_samples': 44, 'subsample': 0.9970297203974366, 'colsample_bytree': 0.9324460763054059, 'reg_alpha': 0.324803213211078, 'reg_lambda': 0.1601835613567248}\n```","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Multioutput","metadata":{}},{"cell_type":"code","source":"# import numpy as np\n# import pandas as pd\n# from copy import deepcopy\n# from sklearn.base import clone\n# from sklearn.decomposition import PCA\n# from sklearn.multioutput import MultiOutputRegressor\n# from sklearn.multioutput import RegressorChain\n\n# # Import the specific libraries\n# from xgboost import XGBRegressor\n# from lightgbm import LGBMRegressor\n# from catboost import CatBoostRegressor\n\n# # ---------------------------------------------------------\n# # 1. The Optimized Multi-Output GPU CV Function\n# # ---------------------------------------------------------\n# def cross_validate_multioutput_gpu(model, train_data, test_data, feature_engine, target_transform='max', n_splits=5):\n#     \"\"\"\n#     Performs Cross Validation using a Multi-Output strategy (Vectorized).\n    \n#     Args:\n#         model: An estimator capable of Multi-Output regression (e.g. XGBoost, MultiOutputRegressor)\n#         train_data: Training DataFrame\n#         test_data: Test DataFrame\n#         feature_engine: A transformer (pipeline or class) with .fit() and .transform()\n#         target_transform: 'log' (np.log1p), 'max' (TARGET_MAX division), or None\n#         n_splits: Number of CV folds\n#     \"\"\"\n    \n#     # 1. Setup Target Max Array (needed for 'max' scaling and broadcasting)\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES].values\n    \n#     # 2. Pre-allocate arrays\n#     # We use a zero-filled array for OOF predictions to fill via indexing\n#     y_pred = np.zeros([len(train_data), len(TARGET_NAMES)], dtype=float)\n#     y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n#     print(f\"Starting CV with model: {model.__class__.__name__}\")\n#     print(f\"Target Transform Strategy: {target_transform}\")\n\n#     for fold in range(n_splits):\n#         # ------------------------------\n#         # Data Preparation\n#         # ------------------------------\n#         # Create masks\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         # Raw Inputs (Features)\n#         X_train_raw = train_data.loc[train_mask, COLUMNS].values\n#         X_valid_raw = train_data.loc[valid_mask, COLUMNS].values\n#         X_test_raw = test_data[COLUMNS].values\n        \n#         # Raw Targets\n#         y_train = train_data.loc[train_mask, TARGET_NAMES].values\n#         # y_valid is used only for metric calculation later, not for training\n#         y_valid = train_data.loc[valid_mask, TARGET_NAMES].values\n\n#         # ------------------------------\n#         # 1) Transform Targets (Vectorized)\n#         # ------------------------------\n#         if target_transform == 'log':\n#             # Log(1+x) handles zeros and stabilizes variance\n#             y_train_proc = np.log1p(y_train)\n#         elif target_transform == 'max':\n#             # Vectorized division\n#             y_train_proc = y_train / target_max_arr\n#         else:\n#             y_train_proc = y_train\n        \n#         # ------------------------------\n#         # 2) Feature Engineering\n#         # ------------------------------\n#         # Fit engine only on training data for this fold to avoid leakage\n#         engine = deepcopy(feature_engine)\n#         engine.fit(X_train_raw)\n        \n#         x_train_eng = engine.transform(X_train_raw)\n#         x_valid_eng = engine.transform(X_valid_raw)\n#         x_test_eng = engine.transform(X_test_raw)\n\n#         # ------------------------------\n#         # 3) Train (Multi-Output)\n#         # ------------------------------\n#         # Clone ensures a fresh model instance per fold\n#         regr = clone(model) \n        \n#         # Fit on (N_samples, N_targets)\n#         # Assuming model supports multi-output natively\n#         regr.fit(x_train_eng, y_train_proc)\n\n#         # ------------------------------\n#         # 4) Predict & Unscale\n#         # ------------------------------\n#         # Predict returns (N_samples, N_targets)\n#         valid_pred_raw = np.array(regr.predict(x_valid_eng))\n#         test_pred_raw = np.array(regr.predict(x_test_eng))\n\n#         # Inverse Transform\n#         if target_transform == 'log':\n#             # Inverse of log1p is expm1\n#             valid_pred = np.expm1(valid_pred_raw)\n#             test_pred = np.expm1(test_pred_raw)\n#         elif target_transform == 'max':\n#             # Vectorized multiplication\n#             valid_pred = valid_pred_raw * target_max_arr\n#             test_pred = test_pred_raw * target_max_arr\n#         else:\n#             valid_pred = valid_pred_raw\n#             test_pred = test_pred_raw\n\n#         # Clip negative predictions (Physical impossibility for biomass/yield)\n#         valid_pred = valid_pred.clip(0)\n#         test_pred = test_pred.clip(0)\n\n#         # Store OOF\n#         y_pred[val_idx] = valid_pred\n        \n#         # Accumulate Test Preds\n#         y_pred_test += test_pred / n_splits\n\n#         # Optional: Print fold score if metric function exists\n#         try:\n#             fold_score = competition_metric(y_valid, valid_pred)\n#             print(f\"  Fold {fold} score: {fold_score:.6f}\")\n#         except NameError:\n#             pass # competition_metric not defined\n            \n#         if fold == 0:\n#              print(f\"  [Fold 0 Debug] Transformed Train Shape: {x_train_eng.shape}\")\n\n#     # Global CV score\n#     try:\n#         full_cv = competition_metric(y_true, y_pred)\n#         print(f\"Full CV Score: {full_cv:.6f}\")\n#     except NameError:\n#         print(\"Done (metric function not found)\")\n\n#     return y_pred, y_pred_test\n\n# def cross_validate_regressor_chain(model, train_data, test_data, feature_engine, target_transform='max', chain_order=None, n_splits=5, random_state=42):\n#     \"\"\"\n#     Performs Cross Validation using a Regressor Chain strategy.\n    \n#     RegressorChain fits a separate model for each target, but uses the predictions \n#     of previous targets in the chain as features for subsequent targets.\n    \n#     Args:\n#         model: A base estimator (e.g. XGBoost, Ridge). NOT a MultiOutputRegressor.\n#         train_data: Training DataFrame\n#         test_data: Test DataFrame\n#         feature_engine: A transformer (pipeline or class) with .fit() and .transform()\n#         target_transform: 'log', 'max', or None\n#         chain_order: Order of targets. None (default order), 'random', or list of indices.\n#         n_splits: Number of CV folds\n#     \"\"\"\n    \n#     # 1. Setup Target Max Array\n#     target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n#     y_true = train_data[TARGET_NAMES].values\n    \n#     # 2. Pre-allocate arrays\n#     y_pred = np.zeros([len(train_data), len(TARGET_NAMES)], dtype=float)\n#     y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n#     print(f\"Starting Chain CV with base model: {model.__class__.__name__}\")\n#     print(f\"Target Transform: {target_transform} | Chain Order: {chain_order}\")\n\n#     for fold in range(n_splits):\n#         # ------------------------------\n#         # Data Preparation\n#         # ------------------------------\n#         train_mask = train_data['fold'] != fold\n#         valid_mask = train_data['fold'] == fold\n#         val_idx = train_data[valid_mask].index\n\n#         # Raw Inputs\n#         X_train_raw = train_data.loc[train_mask, COLUMNS].values\n#         X_valid_raw = train_data.loc[valid_mask, COLUMNS].values\n#         X_test_raw = test_data[COLUMNS].values\n        \n#         # Raw Targets\n#         y_train = train_data.loc[train_mask, TARGET_NAMES].values\n#         y_valid = train_data.loc[valid_mask, TARGET_NAMES].values\n\n#         # ------------------------------\n#         # 1) Transform Targets\n#         # ------------------------------\n#         if target_transform == 'log':\n#             y_train_proc = np.log1p(y_train)\n#         elif target_transform == 'max':\n#             y_train_proc = y_train / target_max_arr\n#         else:\n#             y_train_proc = y_train\n        \n#         # ------------------------------\n#         # 2) Feature Engineering\n#         # ------------------------------\n#         engine = deepcopy(feature_engine)\n#         engine.fit(X_train_raw)\n        \n#         x_train_eng = engine.transform(X_train_raw)\n#         x_valid_eng = engine.transform(X_valid_raw)\n#         x_test_eng = engine.transform(X_test_raw)\n\n#         # ------------------------------\n#         # 3) Train (Regressor Chain)\n#         # ------------------------------\n#         # We clone the base model to ensure a fresh start\n#         base_model = clone(model)\n        \n#         # Create the Chain\n#         # Note: If chain_order is 'random', we need the random_state to be consistent\n#         chain = RegressorChain(base_estimator=base_model, order=chain_order, random_state=random_state)\n        \n#         # Fit the chain on processed targets\n#         # Internally, this fits Model 1, then adds Model 1's preds to X to fit Model 2, etc.\n#         chain.fit(x_train_eng, y_train_proc)\n\n#         # ------------------------------\n#         # 4) Predict & Unscale\n#         # ------------------------------\n#         # Chain.predict automatically handles the feed-forward of predictions\n#         valid_pred_raw = np.array(chain.predict(x_valid_eng))\n#         test_pred_raw = np.array(chain.predict(x_test_eng))\n\n#         # Inverse Transform\n#         if target_transform == 'log':\n#             valid_pred = np.expm1(valid_pred_raw)\n#             test_pred = np.expm1(test_pred_raw)\n#         elif target_transform == 'max':\n#             valid_pred = valid_pred_raw * target_max_arr\n#             test_pred = test_pred_raw * target_max_arr\n#         else:\n#             valid_pred = valid_pred_raw\n#             test_pred = test_pred_raw\n\n#         # Clip negative predictions\n#         valid_pred = valid_pred.clip(0)\n#         test_pred = test_pred.clip(0)\n\n#         # Store OOF\n#         y_pred[val_idx] = valid_pred\n#         y_pred_test += test_pred / n_splits\n\n#         # Optional: Print fold score\n#         try:\n#             fold_score = competition_metric(y_valid, valid_pred)\n#             print(f\"  Fold {fold} score: {fold_score:.6f}\")\n#         except NameError:\n#             pass\n\n#         if fold == 0:\n#              print(f\"  [Fold 0 Debug] Train Shape: {x_train_eng.shape}\")\n\n#     # Global CV score\n#     try:\n#         full_cv = competition_metric(y_true, y_pred)\n#         print(f\"Full CV Score: {full_cv:.6f}\")\n#     except NameError:\n#         print(\"Done (metric function not found)\")\n\n#     return y_pred, y_pred_test\n\n# feat_engine = EmbeddingFeatureEngine(\n#     n_pca_components=0.90, \n#     n_clusters=25, \n#     use_stats=True, \n#     use_similarity=True,\n#     use_anomaly=True,        # Adds Anomaly Score\n#     use_entropy=True,        # Adds Entropy\n#     use_pca_interactions=True # Adds Poly features on Top 5 PCA\n# )\n\n# # ---------------------------------------------------------\n# # 2. Model Definitions\n# # ---------------------------------------------------------\n# best_cat_params = {\n#     'iterations': 1783, \n#     'learning_rate': 0.0633221588945314, \n#     'depth': 4, \n#     'l2_leaf_reg': 0.1312214556803292, \n#     'random_strength': 0.04403178418151252, \n#     'bagging_temperature': 0.9555074383215754\n# }\n# best_cat_params.update({\n#     'loss_function': 'MultiRMSE', \n#     # 'task_type': 'GPU', \n#     'boosting_type': 'Plain', \n#     # 'devices': '0', \n#     'verbose': 0, \n#     'random_state': 42\n# })\n\n# best_xgb_params = {\n#     'n_estimators': 1501, \n#     'learning_rate': 0.024461148923117938, \n#     'max_depth': 3, \n#     'subsample': 0.6905614627569726, \n#     'colsample_bytree': 0.895428293256401, \n#     'reg_alpha': 0.4865138988842402, \n#     'reg_lambda': 0.6015849227570268\n# }\n# best_xgb_params.update({\n#     'tree_method': 'hist', \n#     # 'device': 'cuda', \n#     'n_jobs': -1, \n#     'random_state': 42\n# })\n\n# best_lgbm_params = {\n#     'n_estimators': 1232, \n#     'learning_rate': 0.045467475791811464, \n#     'num_leaves': 32, \n#     'min_child_samples': 38, \n#     'subsample': 0.9389508238313968, \n#     'colsample_bytree': 0.8358504077200445, \n#     'reg_alpha': 0.10126277169074206, \n#     'reg_lambda': 0.1357065010990351\n# }\n# best_lgbm_params.update({\n#     # 'device': 'gpu', \n#     'n_jobs': -1, \n#     'random_state': 42, \n#     'verbose': -1\n# })\n\n# # --- A. XGBoost (Wrapped) ---\n# # XGBoost requires MultiOutputRegressor wrapper for multi-target\n# xgb_model = MultiOutputRegressor(\n#     XGBRegressor(\n#         **best_xgb_params\n#     )\n# )\n\n# # --- B. LightGBM (Wrapped) ---\n# # LightGBM requires MultiOutputRegressor wrapper.\n# # Note: Ensure you have the GPU-compiled version of LightGBM installed.\n# lgbm_model = MultiOutputRegressor(\n#     LGBMRegressor(\n#         **best_lgbm_params\n#     )\n# )\n\n# # --- C. CatBoost (Native) ---\n# # CatBoost supports \"MultiRMSE\" natively. No wrapper needed.\n# # This is usually the fastest option for multi-target on GPU.\n# cat_model = CatBoostRegressor(\n#     **best_cat_params\n# )\n\n# # ---------------------------------------------------------\n# # 3. Usage Example\n# # ---------------------------------------------------------\n# # Assuming 'train' and 'test' pandas DataFrames exist\n# # and TARGET_NAMES / TARGET_MAX / COLUMNS are defined globally\n\n# # 1. Run XGBoost\n# print(\"\\n--- Running XGBoost ---\")\n# oof_xgb, test_xgb = cross_validate_multioutput_gpu(xgb_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# compare_results(oof_xgb, train_siglip_df)\n\n# # 2. Run LightGBM\n# print(\"\\n--- Running LightGBM ---\")\n# oof_lgbm, test_lgbm = cross_validate_multioutput_gpu(lgbm_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# compare_results(oof_lgbm, train_siglip_df)\n\n# # 3. Run CatBoost\n# print(\"\\n--- Running CatBoost ---\")\n# oof_cat, test_cat = cross_validate_multioutput_gpu(cat_model, train_siglip_df, test_siglip_df, feature_engine=feat_engine)\n# compare_results(oof_cat, train_siglip_df)\n\n# print(\"\\n######## Ridge Regression #######\")\n# # ridge_model = MultiOutputRegressor(\n# #     Ridge()\n# # )\n# oof_ridge, pred_test_ri = cross_validate_multioutput_gpu(\n#     Ridge(), \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n# )\n# compare_results(oof_ridge, train_siglip_df)\n\n# print(\"\\n###### Bayesian Ridge Regressor #######\")\n# bayesian_model = MultiOutputRegressor(\n#     BayesianRidge()\n# )\n# oof_bayesian, pred_test_bri = cross_validate_multioutput_gpu(\n#     bayesian_model, \n#     train_siglip_df, test_siglip_df, \n#     feature_engine=feat_engine,\n# )\n# compare_results(oof_bayesian, train_siglip_df)\n\n# # print(\"\\n###### Hist Gradient Boosting Regressor ######\")\n# # hist_model = MultiOutputRegressor(\n# #     HistGradientBoostingRegressor()\n# # )\n# # _, pred_test_hb = cross_validate_regressor_chain(\n# #     HistGradientBoostingRegressor(), \n# #     train_siglip_df, test_siglip_df, \n# #     feature_engine=feat_engine,\n# #     chain_order=chain_order_indices,   # <--- PASS THE ORDER HERE\n# # )\n\n# # print(\"\\n##### Extra Trees Regressor ######\")\n# # et_model = MultiOutputRegressor(\n# #     ExtraTreesRegressor()\n# # )\n# # _, pred_test_et = cross_validate_regressor_chain(\n# #     ExtraTreesRegressor(), \n# #     train_siglip_df, test_siglip_df, \n# #     feature_engine=feat_engine\n# # )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.457619Z","iopub.status.idle":"2025-12-12T16:17:29.458008Z","shell.execute_reply.started":"2025-12-12T16:17:29.457839Z","shell.execute_reply":"2025-12-12T16:17:29.457853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pred_test = (\n    pred_test_gb\n    + pred_test_hb\n    # + pred_test_svr\n    # + pred_test_et\n) / 2\n\n# pred_test = (\n#     pred_test_hb\n#     + pred_test_et\n# ) / 2\n\n# pred_test = (\n#     pred_test_gb\n#     + pred_test_hb\n#     + pred_test_et\n# ) / 3\n\n# pred_test = (\n#     pred_test_ri\n#     + pred_test_gb\n#     + pred_test_hb\n#     + pred_test_et\n# ) / 4\n\n# pred_test = (test_xgb + test_lgbm + test_cat + pred_test_ri + pred_test_bri) / 5\n# pred_test = 0.6*pred_test_ri + 0.15*pred_test_gb + 0.15*pred_test_hb + 0.1*pred_test_et","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.459201Z","iopub.status.idle":"2025-12-12T16:17:29.459525Z","shell.execute_reply.started":"2025-12-12T16:17:29.459392Z","shell.execute_reply":"2025-12-12T16:17:29.459406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df[TARGET_NAMES] = pred_test\ntest_df = post_process_biomass(test_df)\n# test_df['GDM_g'] = test_df['Dry_Green_g'] + test_df['Dry_Clover_g']\n# test_df['Dry_Total_g'] = test_df['GDM_g'] + test_df['Dry_Dead_g']\nsub_df = melt_table(test_df)\nsub_df[['sample_id', 'target']].to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.462561Z","iopub.status.idle":"2025-12-12T16:17:29.462972Z","shell.execute_reply.started":"2025-12-12T16:17:29.462811Z","shell.execute_reply":"2025-12-12T16:17:29.462827Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv(\"submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-12T16:17:29.466032Z","iopub.status.idle":"2025-12-12T16:17:29.466534Z","shell.execute_reply.started":"2025-12-12T16:17:29.466346Z","shell.execute_reply":"2025-12-12T16:17:29.466365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}