{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14018229,"sourceType":"datasetVersion","datasetId":8929818},{"sourceId":288467413,"sourceType":"kernelVersion"},{"sourceId":268942,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":230141,"modelId":251887}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport glob\nfrom pathlib import Path\nimport sys\nfrom tqdm.auto import tqdm\nimport json\nfrom copy import deepcopy\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport math\nimport random\n\nimport torch\nimport torch.nn as nn\nfrom torchvision import transforms\n\nfrom PIL import Image\nimport cv2\n\nfrom transformers import AutoProcessor, AutoImageProcessor, AutoModel, Siglip2Model, Siglip2ImageProcessor, SiglipModel, SiglipImageProcessor\nfrom transformers import AutoModel, AutoTokenizer\nfrom sklearn.model_selection import KFold, GroupKFold\nfrom sklearn.linear_model import LinearRegression, Ridge, Lasso, BayesianRidge\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor, ExtraTreesRegressor\nfrom xgboost import XGBRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.svm import LinearSVR\nfrom sklearn.dummy import DummyRegressor\n\nfrom sklearn.decomposition import PCA\nfrom sklearn import model_selection\nfrom sklearn import preprocessing\n\nfrom dataclasses import dataclass\nfrom typing import Optional, Dict\n\nimport matplotlib.pyplot as plt\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    # pl.seed_everything(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.cuda.manual_seed_all(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n    print('seeding done!!!')\n\ndef flush():\n    import gc\n    gc.collect()\n    if torch.cuda.is_available():\n        torch.cuda.empty_cache()\n        torch.cuda.reset_peak_memory_stats()\n\nseeding(42)\n@dataclass\nclass Config:\n    # Data paths\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    TRAIN_DATA_PATH: Path = DATA_PATH/'train'\n    TEST_DATA_PATH: Path = DATA_PATH/'test'\n\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    seed = 42\n\ncfg = Config()\nseeding(cfg.seed)\ndef pivot_table(df: pd.DataFrame)->pd.DataFrame:\n\n    if 'target' in df.columns.tolist():\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\n# train_df = pd.read_csv(cfg.DATA_PATH/'train.csv')\ntest_df = pd.read_csv(cfg.DATA_PATH/'test.csv')\n# train_df = pivot_table(df=train_df)\ntest_df = pivot_table(df=test_df)\n\n# train_df['image_path'] = train_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\ntest_df['image_path'] = test_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\n# test_df.head()\n\ntrain_df = pd.read_csv(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")\ndef melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    TARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)  # remove folder path, keep filename\n        .str.replace('.jpg', '', regex=False)  # remove extension\n        + '__' + melted['target_name']\n    )\n    \n    return melted[['sample_id', 'image_path', 'target_name', 'target']]\ndef split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    \n    patches = []\n    coords  = []   # (y1, x1, y2, x2)\n    \n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y1 = y\n            x1 = x\n            y2 = y + patch_size\n            x2 = x + patch_size\n            \n            # Pad last patch if needed (very rare with your fixed 1000×2000)\n            patch = image[y1:y2, x1:x2, :]\n            if patch.shape[0] < patch_size or patch.shape[1] < patch_size:\n                pad_h = patch_size - patch.shape[0]\n                pad_w = patch_size - patch.shape[1]\n                patch = np.pad(patch, ((0,pad_h), (0,pad_w), (0,0)), mode='reflect')\n            \n            patches.append(patch)\n            coords.append((y1, x1, y2, x2))\n    \n    return patches, coords\ndef get_model(model_path: str, device: str = 'cpu'):\n    model = AutoModel.from_pretrained(\n        model_path,\n        local_files_only=True\n    )\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    return model.eval().to(device), processor\n\ndino_path = \"/kaggle/input/dinov2/pytorch/giant/1\"\nsiglip_path = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\ndef compute_embeddings(model_path, df, patch_size=520):\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    print(device)\n\n    model, processor = get_model(\n        model_path=model_path, device=device\n    )\n\n    IMAGE_PATHS = []\n    EMBEDDINGS = []\n\n    for i, row in tqdm(df.iterrows(), total=len(df)):\n        img_path = row['image_path']\n        img = cv2.imread(img_path, cv2.IMREAD_COLOR)\n        patches, coords = split_image(img, patch_size=patch_size)\n        images = [Image.fromarray(p).convert(\"RGB\") for p in patches]\n\n        inputs = processor(images=images, return_tensors=\"pt\").to(model.device)\n        with torch.no_grad():\n            if 'siglip' in model_path:\n                features = model.get_image_features(**inputs)\n            elif 'dino' in model_path:\n                features = model(**inputs).pooler_output\n                # patches = model(**inputs).last_hidden_state\n                # features = patches[:, 0, :]\n            else:\n                raise Exception(\"Model should be dino or siglip\")\n        embeds = features.mean(dim=0).detach().cpu().numpy()\n        EMBEDDINGS.append(embeds)\n        IMAGE_PATHS.append(img_path)\n\n    embeddings = np.stack(EMBEDDINGS, axis=0)\n    n_features = embeddings.shape[1]\n    emb_columns = [f\"emb{i+1}\" for i in range(n_features)]\n    emb_df = pd.DataFrame(embeddings, columns=emb_columns)\n    emb_df['image_path'] = IMAGE_PATHS\n    df_final = df.merge(emb_df, on='image_path', how='left')\n    flush()\n    return df_final \n\n# train_siglip_df = compute_embeddings(model_path=siglip_path, df=train_df, patch_size=520)\ntrain_siglip_df = train_df.copy()\ntest_siglip_df = compute_embeddings(model_path=siglip_path, df=test_df, patch_size=520)\n\nflush()\ndef generate_semantic_features(image_embeddings, model_path=siglip_path):\n    \"\"\"\n    Generates 'Concept Scores' by averaging synonyms and calculating biological ratios.\n    \"\"\"\n    print(f\"Loading SigLIP Text Encoder from {model_path}...\")\n    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n    \n    try:\n        model = AutoModel.from_pretrained(model_path).to(device)\n        tokenizer = AutoTokenizer.from_pretrained(model_path)\n    except Exception as e:\n        print(f\"Error loading model: {e}\")\n        return None\n\n    # 1. Define Concept Ensembles (Grouping synonyms reduces noise)\n    concept_groups = {\n        # Quantity Anchors\n        \"bare\": [\"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"],\n        \"sparse\": [\"low density pasture\", \"thin grass\", \"short clipped grass\"],\n        \"medium\": [\"average pasture cover\", \"medium height grass\", \"grazed pasture\"],\n        \"dense\": [\"dense tall pasture\", \"thick grassy volume\", \"high biomass\", \"overgrown vegetation\"],\n        \n        # State Anchors\n        \"green\": [\"lush green vibrant pasture\", \"photosynthesizing leaves\", \"fresh growth\"],\n        \"dead\": [\"dry brown dead grass\", \"yellow straw\", \"senesced material\", \"standing hay\"],\n        \n        # Species Anchors\n        \"clover\": [\"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"],\n        \"grass\": [\"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"],\n        \"weeds\": [\"broadleaf weeds\", \"thistles\", \"non-pasture vegetation\"]\n    }\n    \n    # 2. Encode and Average Prompts for each Concept\n    concept_vectors = {}\n    with torch.no_grad():\n        for name, prompts in concept_groups.items():\n            inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(device)\n            emb = model.get_text_features(**inputs)\n            emb = emb / emb.norm(p=2, dim=-1, keepdim=True)\n            # Average the embeddings of synonyms to get a stable \"Concept Vector\"\n            concept_vectors[name] = emb.mean(dim=0, keepdim=True)\n\n    # 3. Compute Concept Scores\n    if isinstance(image_embeddings, np.ndarray):\n        img_tensor = torch.tensor(image_embeddings, dtype=torch.float32).to(device)\n    else:\n        img_tensor = image_embeddings.to(device)\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n\n    scores = {}\n    for name, vec in concept_vectors.items():\n        # Dot product\n        scores[name] = torch.matmul(img_tensor, vec.T).cpu().numpy().flatten()\n    \n    # 4. Feature Engineering: Explicit Ratios\n    # These help models distinguish between \"High Biomass Dead\" vs \"High Biomass Green\"\n    \n    # Convert dict to DataFrame for easy math\n    df_scores = pd.DataFrame(scores)\n    \n    # A. Greenness Ratio: Green / (Green + Dead)\n    df_scores['ratio_greenness'] = df_scores['green'] / (df_scores['green'] + df_scores['dead'] + 1e-6)\n    \n    # B. Legume Fraction: Clover / (Clover + Grass)\n    df_scores['ratio_clover'] = df_scores['clover'] / (df_scores['clover'] + df_scores['grass'] + 1e-6)\n    \n    # C. Vegetation Cover: (Dense + Medium) / (Bare + Sparse)\n    df_scores['ratio_cover'] = (df_scores['dense'] + df_scores['medium']) / (df_scores['bare'] + df_scores['sparse'] + 1e-6)\n    \n    # D. \"Volume\": Max of density anchors\n    df_scores['max_density'] = df_scores[['bare', 'sparse', 'medium', 'dense']].max(axis=1)\n\n    print(f\"Generated {df_scores.shape[1]} semantic features (Ensembles + Ratios).\")\n    return df_scores.values\nimport numpy as np\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.linear_model import BayesianRidge\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n\nclass SupervisedEmbeddingEngine(BaseEstimator, TransformerMixin):\n    def __init__(self, \n                 n_pca=0.98,  # Slightly higher to keep texture details\n                 n_pls=8,     # Keep 8, it worked well\n                 n_gmm=5,     \n                 random_state=42):\n        \n        self.n_pca = n_pca\n        self.n_pls = n_pls\n        self.n_gmm = n_gmm\n        self.random_state = random_state\n        \n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        # 1. Standard Scaling on IMAGE embeddings only\n        X_scaled = self.scaler.fit_transform(X)\n        \n        # 2. Fit Unsupervised on IMAGES\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        \n        # 3. Fit PLS on IMAGES (Supervised)\n        if y is not None:\n            y_clean = y.values if hasattr(y, 'values') else y\n            self.pls.fit(X_scaled, y_clean)\n            self.pls_fitted_ = True\n        \n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        return self._generate_features(X_scaled, X_semantic)\n\n    def _generate_features(self, X_scaled, X_semantic=None):\n        features = []\n        \n        # A. PCA (Texture/Structure from Images)\n        f_pca = self.pca.transform(X_scaled)\n        features.append(f_pca)\n        \n        # B. PLS (Biomass-correlated signals from Images)\n        if self.pls_fitted_:\n            f_pls = self.pls.transform(X_scaled)\n            features.append(f_pls)\n        \n        # C. GMM (Cluster probs)\n        f_gmm = self.gmm.predict_proba(X_scaled)\n        features.append(f_gmm)\n        \n        # D. Semantic Features (LATE FUSION)\n        # We append them raw. They are already high-level signals.\n        if X_semantic is not None:\n            # Normalize semantic scores relative to themselves to match scale of PCA/PLS\n            # sem_norm = (X_semantic - np.mean(X_semantic, axis=0)) / (np.std(X_semantic, axis=0) + 1e-6)\n            features.append(X_semantic)\n\n        return np.hstack(features)\n\nCOLUMNS = train_df.filter(like=\"emb\").columns\nTARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\nweights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5,\n}\n\nTARGET_MAX = {\n    \"Dry_Green_g\": 157.9836,\n    \"Dry_Dead_g\": 83.8407,\n    \"Dry_Clover_g\": 71.7865,\n    \"GDM_g\": 157.9836,\n    \"Dry_Total_g\": 185.70,\n}\n\ndef competition_metric(y_true, y_pred) -> float:\n    y_weighted = 0\n    for l, label in enumerate(TARGET_NAMES):\n        y_weighted = y_weighted + y_true[:, l].mean() * weights[label]\n\n    ss_res = 0\n    ss_tot = 0\n    for l, label in enumerate(TARGET_NAMES):\n        ss_res = ss_res + ((y_true[:, l] - y_pred[:, l])**2).mean() * weights[label]\n        ss_tot = ss_tot + ((y_true[:, l] - y_weighted)**2).mean() * weights[label]\n\n    return 1 - ss_res / ss_tot\ndef post_process_biomass(df_preds):\n    \"\"\"\n    Enforces physical mass balance constraints on biomass predictions.\n    \n    Constraints enforced:\n    1. Dry_Green_g + Dry_Clover_g = GDM_g\n    2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \n    Method:\n    Uses Orthogonal Projection. It finds the set of values that satisfy\n    the constraints while minimizing the Euclidean distance to the \n    original model predictions.\n    \n    Args:\n        df_preds (pd.DataFrame): DataFrame containing the 5 prediction columns.\n        \n    Returns:\n        pd.DataFrame: A new DataFrame with consistent, non-negative values.\n    \"\"\"\n    # 1. Define the specific order required for the math\n    # We treat the vector x as: [Green, Clover, Dead, GDM, Total]\n    ordered_cols = [\n        \"Dry_Green_g\", \n        \"Dry_Clover_g\", \n        \"Dry_Dead_g\", \n        \"GDM_g\", \n        \"Dry_Total_g\"\n    ]\n    \n    # Check if columns exist\n    if not all(col in df_preds.columns for col in ordered_cols):\n        missing = [c for c in ordered_cols if c not in df_preds.columns]\n        raise ValueError(f\"Input DataFrame is missing columns: {missing}\")\n\n    df_preds[[\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\"]] = (\n        df_preds[[\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\"]].clip(lower=0)\n    )\n    # 2. Extract values in the specific order -> Shape (N_samples, 5)\n    Y = df_preds[ordered_cols].values.T  # Transpose to (5, N) for matrix math\n\n    # 3. Define the Constraint Matrix C\n    # We want Cx = 0\n    # Eq 1: 1*Green + 1*Clover + 0*Dead - 1*GDM + 0*Total = 0\n    # Eq 2: 0*Green + 0*Clover + 1*Dead + 1*GDM - 1*Total = 0\n    C = np.array([\n        [1, 1, 0, -1,  0],\n        [0, 0, 1,  1, -1]\n    ])\n\n    # 4. Calculate Projection Matrix P\n    # P = I - C^T * (C * C^T)^-1 * C\n    # This projects any vector onto the null space of C (the valid subspace)\n    C_T = C.T\n    inv_CCt = np.linalg.inv(C @ C_T)\n    P = np.eye(5) - C_T @ inv_CCt @ C\n\n    # 5. Apply Projection\n    # Y_new = P * Y\n    Y_reconciled = P @ Y\n\n    # 6. Transpose back to (N, 5)\n    Y_reconciled = Y_reconciled.T\n\n    # 7. Post-correction for negatives\n    # Projection can mathematically create negative values (e.g. if Total was predicted 0)\n    # We clip to 0. Note: This might slightly break the sum equality again, \n    # but exact equality with negatives is physically impossible anyway.\n    # Y_reconciled = Y_reconciled.clip(min=0)\n\n    # 8. Create Output DataFrame\n    df_out = df_preds.copy()\n    df_out[ordered_cols] = Y_reconciled\n\n    return df_out\n\n\ndef compare_results(oof, train_data):\n    y_oof_df = pd.DataFrame(oof, columns=TARGET_NAMES) # ensure columns match\n    # 2. Check Score BEFORE Processing\n    raw_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_df.values)\n    print(f\"Raw CV Score: {raw_score:.6f}\")\n    \n    # 3. Apply Post-Processing\n    y_oof_proc = post_process_biomass(y_oof_df)\n    \n    # 4. Check Score AFTER Processing\n    proc_score = competition_metric(train_data[TARGET_NAMES].values, y_oof_proc.values)\n    print(f\"Processed CV Score: {proc_score:.6f}\")\n    \n    print(f\"Improvement: {raw_score - proc_score:.6f}\")\n# --- STEP 1: Generate Semantic Features ---\n# We combine Train and Test to generate features in one go, then split them back.\n# This ensures the text-projections are consistent.\n\n# Concatenate embeddings\nX_all_emb = np.vstack([\n    train_siglip_df[COLUMNS].values, \n    test_siglip_df[COLUMNS].values\n])\n\n# Generate Semantic Probes (Using the function defined in Part 1)\n# Make sure SIGLIP_PATH is correct for your environment\nprint(\"Generating Semantic Features via SigLIP Text Encoder...\")\ntry:\n    all_semantic_scores = generate_semantic_features(X_all_emb, model_path=siglip_path)\n    \n    # Split back into Train and Test\n    n_train = len(train_siglip_df)\n    sem_train_full = all_semantic_scores[:n_train]\n    sem_test_full = all_semantic_scores[n_train:]\n    print(f\"Semantic Features Generated. Train: {sem_train_full.shape}, Test: {sem_test_full.shape}\")\n    \nexcept Exception as e:\n    print(f\"Skipping Semantic Features due to error: {e}\")\n    # Fallback to None if model path is wrong or memory fails\n    sem_train_full = None\n    sem_test_full = None\n# --- STEP 2: Updated Cross-Validation Function ---\ndef cross_validate(model, train_data, test_data, feature_engine, \n                   semantic_train=None, semantic_test=None, # <--- NEW ARGS\n                   target_transform='max', seed=42):\n\n    n_splits = train_data['fold'].nunique()\n    # Setup Targets\n    target_max_arr = np.array([TARGET_MAX[t] for t in TARGET_NAMES], dtype=float)\n    y_true = train_data[TARGET_NAMES]\n    \n    # Setup Storage\n    y_pred = pd.DataFrame(0.0, index=train_data.index, columns=TARGET_NAMES)\n    y_pred_test = np.zeros([len(test_data), len(TARGET_NAMES)], dtype=float)\n\n    for fold in range(n_splits):\n        seeding(seed*(seed//2 + fold))\n        # Create masks\n        train_mask = train_data['fold'] != fold\n        valid_mask = train_data['fold'] == fold\n        val_idx = train_data[valid_mask].index\n\n        # Raw Inputs (Embeddings)\n        X_train_raw = train_data[train_mask][COLUMNS].values\n        X_valid_raw = train_data[valid_mask][COLUMNS].values\n        X_test_raw = test_data[COLUMNS].values\n        \n        # Semantic Inputs (Slicing)\n        # We handle the case where semantic features might be None\n        sem_train_fold = semantic_train[train_mask] if semantic_train is not None else None\n        sem_valid_fold = semantic_train[valid_mask] if semantic_train is not None else None\n        \n        # Raw Targets\n        y_train = train_data[train_mask][TARGET_NAMES].values\n        y_valid = train_data[valid_mask][TARGET_NAMES].values\n\n        # ===========================\n        # 1) TRANSFORM TARGETS\n        # ===========================\n        if target_transform == 'log':\n            y_train_proc = np.log1p(y_train)\n        elif target_transform == 'max':\n            y_train_proc = y_train / target_max_arr\n        else:\n            y_train_proc = y_train\n\n        # ==========================================\n        # 2) FEATURE ENGINEERING\n        # ==========================================\n        engine = deepcopy(feature_engine)\n        \n        # FIT: Now passes y (for PLS/RFE) and Semantic Features\n        engine.fit(X_train_raw, y=y_train_proc, X_semantic=sem_train_fold)\n        \n        # TRANSFORM: Pass Semantic Features\n        x_train_eng = engine.transform(X_train_raw, X_semantic=sem_train_fold)\n        x_valid_eng = engine.transform(X_valid_raw, X_semantic=sem_valid_fold)\n        # For test, we use the full test semantic set\n        x_test_eng = engine.transform(X_test_raw, X_semantic=semantic_test)\n        \n        # ==========================================\n        # 3) TRAIN & PREDICT\n        # ==========================================\n        fold_valid_pred = np.zeros_like(y_valid)\n        fold_test_pred = np.zeros([len(test_data), len(TARGET_NAMES)])\n\n        for k in range(len(TARGET_NAMES)):\n            regr = deepcopy(model)\n            \n            # Fit model\n            regr.fit(x_train_eng, y_train_proc[:, k])\n            \n            # Predict\n            pred_valid_raw = regr.predict(x_valid_eng)\n            pred_test_raw = regr.predict(x_test_eng)\n            \n            # ===========================\n            # 4) INVERSE TRANSFORM\n            # ===========================\n            if target_transform == 'log':\n                pred_valid_inv = np.expm1(pred_valid_raw)\n                pred_test_inv = np.expm1(pred_test_raw)\n            elif target_transform == 'max':\n                pred_valid_inv = (pred_valid_raw * target_max_arr[k])\n                pred_test_inv = (pred_test_raw * target_max_arr[k])\n            else:\n                pred_valid_inv = pred_valid_raw\n                pred_test_inv = pred_test_raw\n\n            fold_valid_pred[:, k] = pred_valid_inv\n            fold_test_pred[:, k] = pred_test_inv\n\n        # Store results\n        y_pred.loc[val_idx] = fold_valid_pred\n        y_pred_test += fold_test_pred / n_splits\n        \n        if fold == 0:\n            print(f\"  [Fold 0 Info] Target: {target_transform}, Feats: {x_train_eng.shape}\")\n\n    full_cv = competition_metric(y_true.values, y_pred.values)\n    print(f\"Full CV Score: {full_cv:.6f}\")\n    \n    return y_pred.values, y_pred_test\n\n# --- STEP 3: Run Models ---\n\n# Initialize the NEW Supervised Engine\nfeat_engine = SupervisedEmbeddingEngine(\n    n_pca=0.80,\n    n_pls=8,             # Supervised signals\n    n_gmm=6,             # Soft clusters\n)\n\nprint(\"\\n###### GradientBoosting Regressor #######\")\ngb_params = {\n    'n_estimators': 1000, 'learning_rate': 0.02, 'max_depth': 3,\n    'subsample': 0.7, 'max_features': 'sqrt', 'min_samples_leaf': 15,\n    'n_iter_no_change': 50, 'validation_fraction': 0.1, 'random_state': 42\n}\noof_gb, pred_test_gb = cross_validate(\n    GradientBoostingRegressor(**gb_params), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_gb, train_siglip_df)\n\nprint(\"\\n###### Hist Gradient Boosting Regressor ######\")\noof_hb, pred_test_hb = cross_validate(\n    HistGradientBoostingRegressor(), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_hb, train_siglip_df)\n\nprint(\"\\n##### CAT Regressor ######\")\noof_cat, pred_test_cat = cross_validate(\n    CatBoostRegressor(verbose=0), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine,\n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full\n)\ncompare_results(oof_cat, train_siglip_df)\n\n\n\nprint(\"\\n######## LGBM #######\")\noof_lgbm, pred_test_lgbm = cross_validate(\n    LGBMRegressor(verbose=-1), \n    train_siglip_df, test_siglip_df, \n    feature_engine=feat_engine, \n    semantic_train=sem_train_full,\n    semantic_test=sem_test_full,\n    target_transform='max')\ncompare_results(oof_lgbm, train_siglip_df)\npred_test = (\n    pred_test_hb\n    + pred_test_gb\n    + pred_test_cat\n    + pred_test_lgbm\n) / 4\ntest_df[TARGET_NAMES] = pred_test\ntest_df = post_process_biomass(test_df)\n# test_df['GDM_g'] = test_df['Dry_Green_g'] + test_df['Dry_Clover_g']\n# test_df['Dry_Total_g'] = test_df['GDM_g'] + test_df['Dry_Dead_g']\nsub_df = melt_table(test_df)\nsub_df[['sample_id', 'target']].to_csv(\"submission.csv\", index=False)\nTARGET = \"Dry_Total_g\"\nTARGET_MAX_VAL = TARGET_MAX[TARGET]\n\nSEEDS = [11, 23, 37, 51, 71, 97]\ndef cv_dry_total(\n    model,\n    train_df,\n    test_df,\n    feature_engine,\n    semantic_train,\n    semantic_test,\n    seed\n):\n    y_true = train_df[TARGET].values\n    y_oof  = np.zeros(len(train_df))\n    y_test = np.zeros(len(test_df))\n\n    n_splits = train_df['fold'].nunique()\n\n    for fold in range(n_splits):\n        seeding(seed * (fold + 19))\n\n        trn_mask = train_df['fold'] != fold\n        val_mask = train_df['fold'] == fold\n        val_idx  = train_df[val_mask].index\n\n        X_trn = train_df.loc[trn_mask, COLUMNS].values\n        X_val = train_df.loc[val_mask, COLUMNS].values\n        X_tst = test_df[COLUMNS].values\n\n        sem_trn = semantic_train[trn_mask]\n        sem_val = semantic_train[val_mask]\n\n        y_trn = train_df.loc[trn_mask, TARGET].values / TARGET_MAX_VAL\n\n        engine = deepcopy(feature_engine)\n        engine.fit(X_trn, y=y_trn.reshape(-1, 1), X_semantic=sem_trn)\n\n        X_trn_e = engine.transform(X_trn, X_semantic=sem_trn)\n        X_val_e = engine.transform(X_val, X_semantic=sem_val)\n        X_tst_e = engine.transform(X_tst, X_semantic=semantic_test)\n\n        reg = deepcopy(model)\n        reg.fit(X_trn_e, y_trn)\n\n        y_oof[val_mask] = reg.predict(X_val_e) * TARGET_MAX_VAL\n        y_test += reg.predict(X_tst_e) * TARGET_MAX_VAL / n_splits\n\n    r2 = 1 - np.mean((y_true - y_oof) ** 2) / np.var(y_true)\n    return r2, y_oof, y_test\ndry_total_model = CatBoostRegressor(\n    iterations=1900,\n    learning_rate=0.045,\n    depth=4,\n    l2_leaf_reg=0.56,\n    random_strength=0.04,\n    bagging_temperature=0.98,\n    loss_function=\"RMSE\",\n    verbose=0,\n    random_state=42\n)\nseed_runs = []\n\nfor seed in SEEDS:\n    print(f\"\\n==== SEED {seed} ====\")\n\n    r2, oof, test_pred = cv_dry_total(\n        model=dry_total_model,\n        train_df=train_siglip_df,\n        test_df=test_siglip_df,\n        feature_engine=feat_engine,\n        semantic_train=sem_train_full,\n        semantic_test=sem_test_full,\n        seed=seed\n    )\n\n    print(f\"Seed {seed} | Dry_Total R2: {r2:.6f}\")\n\n    seed_runs.append({\n        \"seed\": seed,\n        \"r2\": r2,\n        \"oof\": oof,\n        \"test\": test_pred\n    })\nseed_runs = sorted(seed_runs, key=lambda x: x[\"r2\"])\nbest_run = seed_runs[len(seed_runs) // 2]\n\nprint(\"\\nSELECTED SEED:\", best_run[\"seed\"])\nprint(\"Stable R2:\", best_run[\"r2\"])\ntest_df[\"Dry_Total_g\"] = best_run[\"test\"]\n\n# 1. Fill all targets from ensemble\ntest_df[TARGET_NAMES] = pred_test\n\n# 2. LOCK Dry_Total\ntest_df[\"Dry_Total_g\"] = best_run[\"test\"]\n\n# 3. Save Dry_Total copy\n_locked_total = test_df[\"Dry_Total_g\"].values.copy()\n\n# 4. Physics correction\ntest_df = post_process_biomass(test_df)\n\n# 5. RESTORE Dry_Total (IMPORTANT)\ntest_df[\"Dry_Total_g\"] = _locked_total\n# ===============================\n# FINAL SAFE SUBMISSION\n# ===============================\n\ntest_df[TARGET_NAMES] = pred_test\ntest_df[\"Dry_Total_g\"] = best_run[\"test\"]\n\n_locked_total = test_df[\"Dry_Total_g\"].values.copy()\ntest_df = post_process_biomass(test_df)\ntest_df[\"Dry_Total_g\"] = _locked_total\n\nsub_df = melt_table(test_df)\nsub_df[[\"sample_id\", \"target\"]].to_csv(\"submission.csv\", index=False)\npd.read_csv(\"submission.csv\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}