{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":293661155,"sourceType":"kernelVersion"},{"sourceId":268942,"sourceType":"modelInstanceVersion","modelInstanceId":230141,"modelId":251887},{"sourceId":711299,"sourceType":"modelInstanceVersion","modelInstanceId":540408,"modelId":553604},{"sourceId":719660,"sourceType":"modelInstanceVersion","modelInstanceId":547330,"modelId":560139}],"dockerImageVersionId":31236,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":441.06159,"end_time":"2026-01-05T05:15:36.197688","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-01-05T05:08:15.136098","version":"2.6.0"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"5cea4344fed44fffaac71b6fe02ac50b":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"5fd6570613f943ecb07631ffcecddacb":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6b204ec19a394a818e17994aae3263e6":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","background":null,"description_width":"","font_size":null,"text_color":null}},"864a5d2d04374e688a854f72ad0d04d9":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"8f988be66966496085d6e6b8909bf0da":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_864a5d2d04374e688a854f72ad0d04d9","placeholder":"​","style":"IPY_MODEL_5cea4344fed44fffaac71b6fe02ac50b","tabbable":null,"tooltip":null,"value":" 1/1 [00:01&lt;00:00,  1.98s/it]"}},"9a1a60c1824e4af4a02be7dd08d54449":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"af818733f9ec4e22a3500a5692c607b2":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e8a7fcf1682642f9a95f077413387b58","IPY_MODEL_d548382ab4904dd9b1c2742bec3463f0","IPY_MODEL_8f988be66966496085d6e6b8909bf0da"],"layout":"IPY_MODEL_5fd6570613f943ecb07631ffcecddacb","tabbable":null,"tooltip":null}},"b695855014a24a7184902662cc419d1e":{"model_module":"@jupyter-widgets/base","model_module_version":"2.0.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"2.0.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border_bottom":null,"border_left":null,"border_right":null,"border_top":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d372dd276f534b6ba203146ce2f78865":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"2.0.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"d548382ab4904dd9b1c2742bec3463f0":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"ProgressView","bar_style":"success","description":"","description_allow_html":false,"layout":"IPY_MODEL_9a1a60c1824e4af4a02be7dd08d54449","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_d372dd276f534b6ba203146ce2f78865","tabbable":null,"tooltip":null,"value":1}},"e8a7fcf1682642f9a95f077413387b58":{"model_module":"@jupyter-widgets/controls","model_module_version":"2.0.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"2.0.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"2.0.0","_view_name":"HTMLView","description":"","description_allow_html":false,"layout":"IPY_MODEL_b695855014a24a7184902662cc419d1e","placeholder":"​","style":"IPY_MODEL_6b204ec19a394a818e17994aae3263e6","tabbable":null,"tooltip":null,"value":"100%"}}},"version_major":2,"version_minor":0}}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport cv2\nimport torch\nfrom PIL import Image\nfrom tqdm.auto import tqdm\nfrom pathlib import Path\nfrom copy import deepcopy\nfrom dataclasses import dataclass\n\n# Sklearn & Models\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\nfrom sklearn.cross_decomposition import PLSRegression\nfrom sklearn.mixture import GaussianMixture\nfrom sklearn.ensemble import GradientBoostingRegressor, HistGradientBoostingRegressor\nfrom lightgbm import LGBMRegressor\nfrom catboost import CatBoostRegressor\nfrom transformers import AutoModel, AutoImageProcessor, AutoTokenizer\n\n# Suppress warnings\nwarnings.filterwarnings('ignore')\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n# =========================================================================================\n# 1. CONFIGURATION & SEEDING\n# =========================================================================================\n@dataclass\nclass Config:\n    DATA_PATH: Path = Path(\"/kaggle/input/csiro-biomass/\")\n    SPLIT_PATH: Path = Path(\"/kaggle/input/csiro-datasplit/csiro_data_split.csv\")\n    SIGLIP_PATH: str = \"/kaggle/input/google-siglip-so400m-patch14-384/transformers/default/1\"\n    \n    SEED: int = 42\n    DEVICE: str = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n    PATCH_SIZE: int = 520\n    OVERLAP: int = 16\n    \n    # Target definitions\n    TARGET_NAMES = ['Dry_Clover_g', 'Dry_Dead_g', 'Dry_Green_g', 'Dry_Total_g', 'GDM_g']\n    TARGET_MAX = {\n        \"Dry_Clover_g\": 71.7865,\n        \"Dry_Dead_g\": 83.8407,\n        \"Dry_Green_g\": 157.9836,\n        \"Dry_Total_g\": 185.70,\n        \"GDM_g\": 157.9836,\n    }\n\ncfg = Config()\n\ndef seeding(SEED):\n    np.random.seed(SEED)\n    random.seed(SEED)\n    os.environ['PYTHONHASHSEED'] = str(SEED)\n    torch.manual_seed(SEED)\n    if torch.cuda.is_available(): \n        torch.cuda.manual_seed(SEED)\n        torch.backends.cudnn.deterministic = True\n        torch.backends.cudnn.benchmark = False\n\nseeding(cfg.SEED)\n\n# =========================================================================================\n# 2. DATA LOADING & PRE-PROCESSING\n# =========================================================================================\ndef pivot_table(df: pd.DataFrame) -> pd.DataFrame:\n    if 'target' in df.columns.tolist():\n        # Train data\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index=['image_path', 'Sampling_Date', 'State', 'Species', 'Pre_GSHH_NDVI', 'Height_Ave_cm'], \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    else:\n        # Test data\n        df['target'] = 0\n        df_pt = pd.pivot_table(\n            df, \n            values='target', \n            index='image_path', \n            columns='target_name', \n            aggfunc='mean'\n        ).reset_index()\n    return df_pt\n\ndef melt_table(df: pd.DataFrame) -> pd.DataFrame:\n    melted = df.melt(\n        id_vars='image_path',\n        value_vars=cfg.TARGET_NAMES,\n        var_name='target_name',\n        value_name='target'\n    )\n    # Create sample_id matching submission format\n    melted['sample_id'] = (\n        melted['image_path']\n        .str.replace(r'^.*/', '', regex=True)\n        .str.replace('.jpg', '', regex=False)\n        + '__' + melted['target_name']\n    )\n    return melted[['sample_id', 'target']]\n\ndef post_process_biomass(df_preds):\n    \"\"\"\n    Derive GDM_g and Dry_Total_g from primary predictions.\n    IMPORTANT: Keeps Dry_Clover_g fixed at 0.0 and does NOT modify Dry_Green_g or Dry_Dead_g.\n    \"\"\"\n    ordered_cols = [\"Dry_Green_g\", \"Dry_Clover_g\", \"Dry_Dead_g\", \"GDM_g\", \"Dry_Total_g\"]\n    \n    # Ensure cols exist\n    for c in ordered_cols:\n        if c not in df_preds.columns:\n            df_preds[c] = 0.0\n    \n    df_out = df_preds.copy()\n    \n    # Keep Dry_Clover_g fixed at 0.0 (should already be 0.0 from model prediction)\n    df_out['Dry_Clover_g'] = 0.0\n    \n    # Keep Dry_Green_g and Dry_Dead_g as predicted (NO MODIFICATION)\n    # Only derive GDM_g and Dry_Total_g\n    df_out['GDM_g'] = df_out['Dry_Green_g'] + df_out['Dry_Clover_g']\n    df_out['Dry_Total_g'] = df_out['GDM_g'] + df_out['Dry_Dead_g']\n    \n    # Clip only derived targets to non-negative\n    df_out['GDM_g'] = df_out['GDM_g'].clip(lower=0.0)\n    df_out['Dry_Total_g'] = df_out['Dry_Total_g'].clip(lower=0.0)\n    \n    return df_out\n\nprint(\"Loading Data...\")\n# Load Train (Metadata Split)\ntrain_df = pd.read_csv(cfg.SPLIT_PATH) \n\n# --- FIX: Remove pre-existing embedding columns to prevent duplication ---\ncols_to_keep = [c for c in train_df.columns if not c.startswith('emb')]\ntrain_df = train_df[cols_to_keep]\n# -----------------------------------------------------------------------\n\n# Ensure train paths match local environment\nif not str(train_df['image_path'].iloc[0]).startswith('/'):\n     train_df['image_path'] = train_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / 'train' / os.path.basename(p)))\n\n# Load Test\ntest_df_raw = pd.read_csv(cfg.DATA_PATH / 'test.csv')\ntest_df = pivot_table(test_df_raw)\ntest_df['image_path'] = test_df['image_path'].apply(lambda p: str(cfg.DATA_PATH / p))\n\n# =========================================================================================\n# 3. FEATURE EXTRACTION: SIGLIP IMAGE EMBEDDINGS\n# =========================================================================================\ndef split_image(image, patch_size=520, overlap=16):\n    h, w, c = image.shape\n    stride = patch_size - overlap\n    patches = []\n    for y in range(0, h, stride):\n        for x in range(0, w, stride):\n            y2 = min(y + patch_size, h)\n            x2 = min(x + patch_size, w)\n            y1 = max(0, y2 - patch_size) # Ensure fixed size\n            x1 = max(0, x2 - patch_size)\n            patch = image[y1:y2, x1:x2, :]\n            patches.append(patch)\n    return patches\n\ndef compute_embeddings(model_path, df):\n    print(f\"Computing Embeddings for {len(df)} images...\")\n    model = AutoModel.from_pretrained(model_path, local_files_only=True).eval().to(cfg.DEVICE)\n    processor = AutoImageProcessor.from_pretrained(model_path)\n    \n    EMBEDDINGS = []\n    \n    for _, row in tqdm(df.iterrows(), total=len(df)):\n        try:\n            img = cv2.imread(row['image_path'])\n            if img is None: raise ValueError(\"Image not found\")\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n            patches = split_image(img, patch_size=cfg.PATCH_SIZE, overlap=cfg.OVERLAP)\n            images = [Image.fromarray(p) for p in patches]\n            \n            # Batch process patches\n            inputs = processor(images=images, return_tensors=\"pt\").to(cfg.DEVICE)\n            with torch.no_grad():\n                features = model.get_image_features(**inputs)\n            \n            # Average pooling of patches\n            avg_embed = features.mean(dim=0).cpu().numpy()\n            EMBEDDINGS.append(avg_embed)\n        except Exception as e:\n            print(f\"Error processing {row['image_path']}: {e}\")\n            # Fallback zero embedding\n            EMBEDDINGS.append(np.zeros(1152))\n        \n    torch.cuda.empty_cache()\n    return np.stack(EMBEDDINGS)\n\n# Compute Features\ntrain_embeddings = compute_embeddings(cfg.SIGLIP_PATH, train_df)\ntest_embeddings = compute_embeddings(cfg.SIGLIP_PATH, test_df)\n\n# Create Feature DataFrames\nemb_cols = [f\"emb{i}\" for i in range(train_embeddings.shape[1])]\ntrain_feat_df = pd.concat([train_df, pd.DataFrame(train_embeddings, columns=emb_cols)], axis=1)\ntest_feat_df = pd.concat([test_df, pd.DataFrame(test_embeddings, columns=emb_cols)], axis=1)\n\n# Double check column counts\nprint(f\"Train Features Shape: {train_feat_df.shape}\")\nprint(f\"Test Features Shape: {test_feat_df.shape}\")\n\n# =========================================================================================\n# 4. FEATURE EXTRACTION: SEMANTIC FEATURES (TEXT PROBING)\n# =========================================================================================\ndef generate_semantic_features(image_embeddings_np, model_path):\n    print(\"Generating Semantic Features...\")\n    model = AutoModel.from_pretrained(model_path).to(cfg.DEVICE)\n    tokenizer = AutoTokenizer.from_pretrained(model_path)\n    \n    # Anchors\n    concept_groups = {\n        \"bare\": [\"bare soil\", \"dirt ground\", \"sparse vegetation\", \"exposed earth\"],\n        \"sparse\": [\"low density pasture\", \"thin grass\", \"short clipped grass\"],\n        \"medium\": [\"average pasture cover\", \"medium height grass\", \"grazed pasture\"],\n        \"dense\": [\"dense tall pasture\", \"thick grassy volume\", \"high biomass\", \"overgrown vegetation\"],\n        \"green\": [\"lush green vibrant pasture\", \"photosynthesizing leaves\", \"fresh growth\"],\n        \"dead\": [\"dry brown dead grass\", \"yellow straw\", \"senesced material\", \"standing hay\"],\n        \"clover\": [\"white clover\", \"trifolium repens\", \"broadleaf legume\", \"clover flowers\"],\n        \"grass\": [\"ryegrass\", \"blade-like leaves\", \"fescue\", \"grassy sward\"]\n    }\n    \n    # Encode Concepts\n    concept_vectors = {}\n    with torch.no_grad():\n        for name, prompts in concept_groups.items():\n            inputs = tokenizer(prompts, padding=\"max_length\", return_tensors=\"pt\").to(cfg.DEVICE)\n            emb = model.get_text_features(**inputs)\n            emb = emb / emb.norm(p=2, dim=-1, keepdim=True)\n            concept_vectors[name] = emb.mean(dim=0, keepdim=True)\n            \n    # Compute Scores\n    img_tensor = torch.tensor(image_embeddings_np, dtype=torch.float32).to(cfg.DEVICE)\n    img_tensor = img_tensor / img_tensor.norm(p=2, dim=-1, keepdim=True)\n    \n    scores = {}\n    for name, vec in concept_vectors.items():\n        scores[name] = torch.matmul(img_tensor, vec.T).cpu().numpy().flatten()\n        \n    df_scores = pd.DataFrame(scores)\n    # Ratios\n    df_scores['ratio_greenness'] = df_scores['green'] / (df_scores['green'] + df_scores['dead'] + 1e-6)\n    df_scores['ratio_clover'] = df_scores['clover'] / (df_scores['clover'] + df_scores['grass'] + 1e-6)\n    df_scores['ratio_cover'] = (df_scores['dense'] + df_scores['medium']) / (df_scores['bare'] + df_scores['sparse'] + 1e-6)\n    \n    torch.cuda.empty_cache()\n    return df_scores.values\n\n# Combine for semantic generation to ensure consistency\nall_emb = np.vstack([train_embeddings, test_embeddings])\nall_semantic = generate_semantic_features(all_emb, cfg.SIGLIP_PATH)\n\nsem_train = all_semantic[:len(train_df)]\nsem_test = all_semantic[len(train_df):]\n\n# =========================================================================================\n# 5. SUPERVISED EMBEDDING ENGINE\n# =========================================================================================\nclass SupervisedEmbeddingEngine:\n    def __init__(self, n_pca=0.80, n_pls=8, n_gmm=6, random_state=42):\n        self.scaler = StandardScaler()\n        self.pca = PCA(n_components=n_pca, random_state=random_state)\n        self.pls = PLSRegression(n_components=n_pls, scale=False)\n        self.gmm = GaussianMixture(n_components=n_gmm, covariance_type='diag', random_state=random_state)\n        self.pls_fitted_ = False\n\n    def fit(self, X, y=None, X_semantic=None):\n        # Scale\n        X_scaled = self.scaler.fit_transform(X)\n        \n        # Unsupervised\n        self.pca.fit(X_scaled)\n        self.gmm.fit(X_scaled)\n        \n        # Supervised\n        if y is not None:\n            self.pls.fit(X_scaled, y)\n            self.pls_fitted_ = True\n        return self\n\n    def transform(self, X, X_semantic=None):\n        X_scaled = self.scaler.transform(X)\n        \n        features = [self.pca.transform(X_scaled)]\n        \n        if self.pls_fitted_:\n            features.append(self.pls.transform(X_scaled))\n            \n        features.append(self.gmm.predict_proba(X_scaled))\n        \n        if X_semantic is not None:\n            # Normalize semantic\n            sem_norm = (X_semantic - np.mean(X_semantic, axis=0)) / (np.std(X_semantic, axis=0) + 1e-6)\n            features.append(sem_norm)\n            \n        return np.hstack(features)\n\n# =========================================================================================\n# 6. TRAINING & INFERENCE (5-FOLD CV)\n# =========================================================================================\ndef cross_validate_predict(model_cls, model_params, train_data, test_data, sem_tr, sem_te, feature_engine):\n    target_max_arr = np.array([cfg.TARGET_MAX[t] for t in cfg.TARGET_NAMES], dtype=float)\n    y_pred_test_accum = np.zeros([len(test_data), len(cfg.TARGET_NAMES)], dtype=float)\n    \n    # Ensure n_splits is integer\n    n_splits = int(train_data['fold'].nunique())\n    \n    # Pre-extract raw columns to avoid indexing overhead\n    # Force float32 to save memory and ensure compatibility\n    X_train_full = train_data[emb_cols].values.astype(np.float32)\n    X_test_raw = test_data[emb_cols].values.astype(np.float32)\n    y_train_full = train_data[cfg.TARGET_NAMES].values.astype(np.float32)\n    \n    for fold in range(n_splits):\n        print(f\"Processing Fold {fold}...\")\n        # Split\n        train_mask = train_data['fold'] != fold\n        \n        X_tr = X_train_full[train_mask]\n        y_tr = y_train_full[train_mask] / target_max_arr # Max Scaling\n        \n        sem_tr_fold = sem_tr[train_mask]\n        \n        # Feature Engineering (Fit on fold train)\n        engine = deepcopy(feature_engine)\n        engine.fit(X_tr, y=y_tr, X_semantic=sem_tr_fold)\n        \n        x_tr_eng = engine.transform(X_tr, X_semantic=sem_tr_fold)\n        x_te_eng = engine.transform(X_test_raw, X_semantic=sem_te)\n        \n        # Train & Predict per target\n        fold_test_pred = np.zeros([len(test_data), len(cfg.TARGET_NAMES)])\n        \n        for k in range(len(cfg.TARGET_NAMES)):\n            target_name = cfg.TARGET_NAMES[k]\n            \n            # Dry_Clover_g: Always predict 0.0 (no training needed)\n            if target_name == 'Dry_Clover_g':\n                fold_test_pred[:, k] = 0.0\n            else:\n                # Train and predict for other targets\n                model = model_cls(**model_params)\n                model.fit(x_tr_eng, y_tr[:, k])\n                pred_raw = model.predict(x_te_eng)\n                fold_test_pred[:, k] = pred_raw * target_max_arr[k] # Inverse Scale\n            \n        y_pred_test_accum += fold_test_pred\n        \n    return y_pred_test_accum / n_splits\n\n# Model Parameters (Optimized)\nparams_cat = {\n    'iterations': 1900, 'learning_rate': 0.045, 'depth': 4, 'l2_leaf_reg': 0.56, \n    'random_strength': 0.045, 'bagging_temperature': 0.98, 'verbose': 0, 'random_state': 42,\n    'allow_writing_files': False\n}\nparams_xgb = { # Using GradientBoostingRegressor as proxy\n    'n_estimators': 1354, 'learning_rate': 0.010, 'max_depth': 3, 'subsample': 0.60, \n    'random_state': 42\n}\nparams_lgbm = {\n    'n_estimators': 807, 'learning_rate': 0.014, 'num_leaves': 48, 'min_child_samples': 19, \n    'subsample': 0.745, 'colsample_bytree': 0.745, 'reg_alpha': 0.21, 'reg_lambda': 3.78,\n    'verbose': -1, 'random_state': 42\n}\nparams_hist = {\n    'max_iter': 300, 'learning_rate': 0.05, 'max_depth': None, 'l2_regularization': 0.44,\n    'random_state': 42\n}\n\nfeat_engine = SupervisedEmbeddingEngine(n_pca=0.80, n_pls=8, n_gmm=6)\n\nprint(\"Training & Inferring Models...\")\n\n# 1. HistGradientBoosting\nprint(\"Model: HistGradientBoosting\")\npred_hist = cross_validate_predict(\n    HistGradientBoostingRegressor, params_hist, \n    train_feat_df, test_feat_df, sem_train, sem_test, feat_engine\n)\n\n# 2. GradientBoosting\nprint(\"Model: GradientBoosting\")\npred_gb = cross_validate_predict(\n    GradientBoostingRegressor, params_xgb, \n    train_feat_df, test_feat_df, sem_train, sem_test, feat_engine\n)\n\n# 3. CatBoost\nprint(\"Model: CatBoost\")\npred_cat = cross_validate_predict(\n    CatBoostRegressor, params_cat, \n    train_feat_df, test_feat_df, sem_train, sem_test, feat_engine\n)\n\n# 4. LightGBM\nprint(\"Model: LightGBM\")\npred_lgbm = cross_validate_predict(\n    LGBMRegressor, params_lgbm, \n    train_feat_df, test_feat_df, sem_train, sem_test, feat_engine\n)\n\n# =========================================================================================\n# 7. ENSEMBLING & SUBMISSION\n# =========================================================================================\nprint(\"Ensembling and Post-processing...\")\n# Simple Average Ensemble\nfinal_pred = (pred_hist + pred_gb + pred_cat + pred_lgbm) / 4.0\n\n# Assign to dataframe\ntest_feat_df[cfg.TARGET_NAMES] = final_pred\n\n# Post-process (Mass Balance)\ntest_processed = post_process_biomass(test_feat_df)\n\n# Create Submission File\nsub_df = melt_table(test_processed)\noutput_path = \"submission_siglip.csv\"\nsub_df.to_csv(output_path, index=False)\n\nprint(f\"✓ Siglip submission generated: {output_path}\")\nprint(sub_df.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:36:10.769899Z","iopub.execute_input":"2026-01-19T15:36:10.7706Z","iopub.status.idle":"2026-01-19T15:51:13.119413Z","shell.execute_reply.started":"2026-01-19T15:36:10.770568Z","shell.execute_reply":"2026-01-19T15:51:13.118548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%writefile csiro_infer.py\n\nfrom PIL import Image\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold, GroupKFold, StratifiedGroupKFold\nfrom tqdm.auto import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.optim import AdamW\nfrom transformers import get_cosine_schedule_with_warmup\n\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms as T\n\nimport os\nfrom pathlib import Path\nimport timm\nimport warnings \n\nwarnings.filterwarnings('ignore')\ntqdm.pandas()\n\nclass RegressionDataset(Dataset):\n    def __init__(self, data, transform=None):\n        self.data = data\n        self.transform = transform\n    def __len__(self):\n        return self.data.shape[0]\n\n    def __getitem__(self, idx):\n        item = self.data.iloc[idx]\n        image = item.image\n        targets = [item['Dry_Green_g'], item['Dry_Clover_g'], item['Dry_Dead_g']]\n        width, height = image.size\n        mid_point = width // 2\n        left_image = image.crop((0, 0, mid_point, height))\n        right_image = image.crop((mid_point, 0, width, height))\n\n        if self.transform is not None:\n            left_image = self.transform(left_image)\n            right_image = self.transform(right_image)\n\n        return left_image, right_image, targets\n\ndef get_test_dataloaders(data, image_size, batch_size):\n    res = []\n    for trans in [None, T.RandomHorizontalFlip(p=1.0), T.RandomVerticalFlip(p=1.0)]:\n        transform = T.Compose([\n            T.Resize(image_size),\n            T.ToTensor(),\n            T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n        ])\n        if trans:\n            transform = T.Compose([\n                T.Resize(image_size),\n                trans,\n                T.ToTensor(),\n                T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n            ])\n        dataset = RegressionDataset(data, transform=transform)\n        res.append(DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=4))\n    return res\n\n\nclass FiLM(nn.Module):\n    def __init__(self, feat_dim):\n        super().__init__()\n        self.mlp = nn.Sequential(\n            nn.Linear(feat_dim, feat_dim // 2), \n            nn.ReLU(inplace=True), \n            nn.Linear(feat_dim // 2, feat_dim * 2)\n        )\n\n    def forward(self, context):\n        gamma_beta = self.mlp(context)\n        return torch.chunk(gamma_beta, 2, dim=1)\n\nclass CSIROModelRegressor(nn.Module):\n    def __init__(self, model_name, pretrained=True, num_classes=3, dropout=0.0, freeze_backbone=False):\n        super().__init__()\n        self.backbone = timm.create_model(model_name, pretrained=pretrained, num_classes=0, global_pool='avg')\n\n        self.film = FiLM(self.backbone.num_features)\n\n        self.dropout = nn.Dropout(dropout)\n\n        def make_head():\n            return nn.Sequential(\n                nn.Linear(self.backbone.num_features * 2, 8),\n                nn.ReLU(inplace=True),\n                nn.Dropout(dropout),\n                nn.Linear(8, 1),\n            )\n\n        self.head_green = make_head()\n        self.head_clover = make_head()\n        self.head_dead = make_head()\n\n        self.softplus = nn.Softplus(beta=1.0)\n\n        if freeze_backbone:\n            for p in self.backbone.parameters():\n                p.requires_grad = False\n\n\n    def forward(self, left_img, right_img):\n        left_feat = self.backbone(left_img)\n        right_feat = self.backbone(right_img)\n\n        context = (left_feat + right_feat) / 2\n        gamma, beta = self.film(context)\n\n        left_feat_modulated = left_feat * (1 + gamma) + beta\n        right_feat_modulated = right_feat * (1 + gamma) + beta\n\n        combined = torch.cat([left_feat_modulated, right_feat_modulated], dim=1)\n\n        green = self.softplus(self.head_green(combined))   \n        clover = self.softplus(self.head_clover(combined))\n        dead = self.softplus(self.head_dead(combined)) \n\n        logits = torch.cat([green, clover, dead], dim=1)\n\n        return logits\n\ndef predict(model, dataloader, device):\n    model.to(device)\n    model.eval()\n\n    all_outputs = []\n    with torch.no_grad():\n        for left_images, right_images, targets in dataloader:\n            left_images = left_images.to(device)\n            right_images = right_images.to(device)\n\n            outputs = model(left_images, right_images)\n            all_outputs.append(outputs.detach().cpu())\n\n    outputs = torch.cat(all_outputs).numpy()\n    return outputs\n\ndef predict_loaders(model, dataloaders, device):\n    all_outputs = []\n    for dataloader in dataloaders:\n        outputs = predict(model, dataloader, device)\n        all_outputs.append(outputs)\n    avg_outputs = np.mean(all_outputs, axis=0)\n    return avg_outputs\n\ndef predict_folds(dataloaders,models_dir):\n    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n\n    all_preds = []\n    for model_file in Path(models_dir).glob('*.pth'):\n        model = CSIROModelRegressor(CFG.MODEL_NAME, pretrained=False, num_classes=3)\n        model.load_state_dict(torch.load(model_file))\n        preds = predict_loaders(model, dataloaders, device)\n        all_preds.append(preds)\n\n    avg_preds = np.mean(all_preds, axis=0)\n    return avg_preds\n\n\nclass CFG:\n    DATA_PATH=\"/kaggle/input/csiro-biomass/\"\n    TEST_DATA_PATH=\"/kaggle/input/csiro-biomass/test.csv\"\n    MODEL_NAME=\"vit_large_patch16_dinov3_qkvb\"\n    MODELS_DIR ='/kaggle/input/modelv3/pytorch/default/1/models_retrained'\n    IMG_SIZE=(512,512)\n\n\ntest_df = pd.read_csv(CFG.TEST_DATA_PATH)\n\ntest_df['target'] = 0.0\ntest_df[['sample_id_prefix', 'sample_id_suffix']] = test_df.sample_id.str.split('__', expand=True)\n\ntest_data_df = test_df.groupby(['sample_id_prefix', 'image_path']).apply(lambda df: df.set_index('target_name').target)\ntest_data_df.reset_index(inplace=True)\ntest_data_df.columns.name = None\n\ntest_data_df['image'] = test_data_df.image_path.progress_apply(lambda path: Image.open(CFG.DATA_PATH + path).convert('RGB'))\n\ntest_loaders = get_test_dataloaders(test_data_df, CFG.IMG_SIZE, 32)\npreds = predict_folds(test_loaders,models_dir=CFG.MODELS_DIR)\n\n\n\ntest_data_df[['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g']] = preds\ntest_data_df['Dry_Green_g'] = test_data_df['Dry_Green_g'].apply(lambda x: 0.0 if x < 0.09 else x)\ntest_data_df['Dry_Clover_g'] = test_data_df['Dry_Clover_g'].apply(lambda x: 0.0 if x < 0.09 else x)\ntest_data_df['Dry_Dead_g'] = test_data_df['Dry_Dead_g'].apply(lambda x: 0.0 if x < 0.09 else x)\ntest_data_df['GDM_g'] = test_data_df.Dry_Green_g + test_data_df.Dry_Clover_g\ntest_data_df['Dry_Total_g'] = test_data_df.GDM_g + test_data_df.Dry_Dead_g\n\ncols = [ 'Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g' , 'Dry_Total_g']\nsub_df = test_data_df.set_index('sample_id_prefix')[cols].stack()\nsub_df = sub_df.reset_index()\nsub_df.columns = ['sample_id_prefix', 'target_name', 'target']\n\nsub_df['sample_id'] = sub_df.sample_id_prefix + '__' + sub_df.target_name\n\ncols = ['sample_id', 'target']\nsub_df[cols].to_csv('submission_dinov2026.csv', index=False)\n\nprint(sub_df[cols])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:51:13.121062Z","iopub.execute_input":"2026-01-19T15:51:13.121388Z","iopub.status.idle":"2026-01-19T15:51:13.129666Z","shell.execute_reply.started":"2026-01-19T15:51:13.12134Z","shell.execute_reply":"2026-01-19T15:51:13.128864Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!python csiro_infer.py","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:51:13.130614Z","iopub.execute_input":"2026-01-19T15:51:13.130844Z","iopub.status.idle":"2026-01-19T15:52:59.301544Z","shell.execute_reply.started":"2026-01-19T15:51:13.130825Z","shell.execute_reply":"2026-01-19T15:52:59.300784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n# Weights for the ensemble (Must sum to 1.0)\nW_SIGLIP = 0.35\nW_DINO   = 0.65\n\nFILES = {\n    'siglip': 'submission_siglip.csv',\n    'dino':   'submission_dinov2026.csv'\n}\n\nOUTPUT_FILE = 'submission72.csv'\n\n# Target definitions required for Mass Balance\nALL_TARGETS = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n\n# =============================================================================\n# HELPER FUNCTIONS\n# =============================================================================\n\ndef enforce_mass_balance(df_wide, fixed_clover=None):\n    \"\"\"\n    Applies Orthogonal Projection to enforce biological constraints:\n    1. Dry_Green_g + Dry_Clover_g = GDM_g\n    2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \n    If fixed_clover is True, Dry_Clover_g is kept fixed and only other targets are adjusted.\n    \"\"\"\n    # 1. Ensure columns are in the specific order for the matrix math\n    ordered_cols = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n    \n    # Extract values: Shape (5, N_samples)\n    Y = df_wide[ordered_cols].values.T\n    \n    if fixed_clover:\n        # Keep Dry_Clover_g fixed, adjust only other targets\n        # We have: Green + Clover_fixed = GDM, so GDM = Green + Clover_fixed\n        # And: GDM + Dead = Total, so Total = GDM + Dead = Green + Clover_fixed + Dead\n        \n        # Note: In this specific pipeline, we also want to respect the adjusted Dry_Dead_g \n        # from the heuristics if possible, but the standard mass balance logic here \n        # recalculates GDM and Total based on components.\n        \n        clover_fixed = Y[1, :].copy()  # Dry_Clover_g is index 1\n        \n        # Adjust: GDM = Green + Clover_fixed\n        Y[3, :] = Y[0, :] + clover_fixed  # GDM_g\n        \n        # Adjust: Total = GDM + Dead\n        Y[4, :] = Y[3, :] + Y[2, :]  # Dry_Total_g\n        \n        Y_reconciled = Y\n    else:\n        # Standard projection if not fixing specific columns\n        C = np.array([\n            [1, 1, 0, -1,  0],\n            [0, 0, 1,  1, -1]\n        ])\n        C_T = C.T\n        try:\n            inv_CCt = np.linalg.inv(C @ C_T)\n            P = np.eye(5) - C_T @ inv_CCt @ C\n            Y_reconciled = P @ Y\n        except np.linalg.LinAlgError:\n            return df_wide\n\n    # 5. Transpose back to (N_samples, 5) and clip negatives\n    Y_reconciled = Y_reconciled.T\n    Y_reconciled = np.maximum(0, Y_reconciled) \n    \n    # 6. Update DataFrame\n    df_out = df_wide.copy()\n    df_out[ordered_cols] = Y_reconciled\n    \n    return df_out\n\ndef apply_heuristics(row):\n    \"\"\"\n    Applies the advanced preprocessing/post-processing rules \n    found in the competition inference notebook.\n    \"\"\"\n    target_name = row['target_name']\n    prediction = row['target']\n    \n    if target_name == \"Dry_Clover_g\":\n        # Scale down Clover\n        prediction = prediction * 0.8\n        \n    elif target_name == \"Dry_Dead_g\":\n        # Conditional scaling for Dead grass\n        if prediction > 20:\n            prediction *= 1.1\n        elif prediction < 10:\n            prediction *= 0.9\n            \n    return prediction\n\ndef robust_ensemble(file_paths, weights):\n    print(f\"--- Starting Ensemble ---\")\n    print(f\"Weights: {weights}\")\n    \n    dfs = []\n    for name, path in file_paths.items():\n        if not os.path.exists(path):\n            raise FileNotFoundError(f\"Missing file: {path}\")\n        \n        # Read and sort\n        df = pd.read_csv(path).sort_values('sample_id').reset_index(drop=True)\n        dfs.append(df)\n        print(f\"Loaded {name}: {len(df)} rows\")\n\n    # 1. Check alignment\n    base_ids = dfs[0]['sample_id']\n    if not all(df['sample_id'].equals(base_ids) for df in dfs[1:]):\n        raise ValueError(\"Sample IDs do not match between submission files!\")\n\n    # 2. Split by target_name\n    dfs_split = []\n    for df in dfs:\n        df_split = df.copy()\n        df_split[['image_id', 'target_name']] = df_split['sample_id'].str.rsplit('__', n=1, expand=True)\n        dfs_split.append(df_split)\n    \n    dino_idx = list(file_paths.keys()).index('dino')\n    siglip_idx = list(file_paths.keys()).index('siglip')\n    dino_df = dfs_split[dino_idx]\n    siglip_df = dfs_split[siglip_idx]\n    \n    # 3. Aggregation Strategy\n    ensemble_results = []\n    \n    for target in ALL_TARGETS:\n        dino_target = dino_df[dino_df['target_name'] == target].copy()\n        siglip_target = siglip_df[siglip_df['target_name'] == target].copy()\n        \n        if target == 'Dry_Clover_g':\n            # Use DINO only for Clover\n            print(f\"Using DINO-only for {target}\")\n            ensemble_results.append(dino_target[['sample_id', 'image_id', 'target_name', 'target']])\n        else:\n            # Weighted average for others\n            merged = pd.merge(\n                dino_target[['sample_id', 'image_id', 'target_name', 'target']].rename(columns={'target': 'dino_pred'}),\n                siglip_target[['sample_id', 'target']].rename(columns={'target': 'siglip_pred'}),\n                on='sample_id'\n            )\n            \n            w_vec = np.array([weights['dino'], weights['siglip']])\n            w_vec = w_vec / w_vec.sum()\n            \n            merged['target'] = merged['dino_pred'] * w_vec[0] + merged['siglip_pred'] * w_vec[1]\n            ensemble_results.append(merged[['sample_id', 'image_id', 'target_name', 'target']])\n    \n    # 4. Combine all targets\n    ensemble_df = pd.concat(ensemble_results, ignore_index=True)\n    print(\"Weighted average complete.\")\n\n    # =========================================================================\n    # NEW STEP: APPLY COMPETITION HEURISTICS\n    # =========================================================================\n    print(\"Applying Advanced Heuristics (Scaling Clover and Dead)...\")\n    # We apply this row-by-row or vectorized. \n    ensemble_df['target'] = ensemble_df.apply(apply_heuristics, axis=1)\n\n    # 5. Prepare for Mass Balance (Convert Long -> Wide)\n    wide_df = ensemble_df.pivot(index='image_id', columns='target_name', values='target').reset_index()\n\n    # 6. Apply Mass Balance\n    # Since we modified Clover and Dead using heuristics, we pass fixed_clover=True\n    # so the mass balance recalculates GDM and Total to match the new heuristics.\n    print(\"Applying Mass Balance Constraints...\")\n    wide_balanced = enforce_mass_balance(wide_df, fixed_clover=True)\n\n    # 7. Convert back (Wide -> Long)\n    long_balanced = wide_balanced.melt(\n        id_vars='image_id', \n        value_vars=ALL_TARGETS,\n        var_name='target_name',\n        value_name='target'\n    )\n\n    # Reconstruct sample_id\n    long_balanced['sample_id'] = long_balanced['image_id'] + '__' + long_balanced['target_name']\n\n    # 8. Final Formatting\n    final_submission = long_balanced[['sample_id', 'target']].sort_values('sample_id').reset_index(drop=True)\n\n    return final_submission\n\n# =============================================================================\n# EXECUTION\n# =============================================================================\nif __name__ == \"__main__\":\n    \n    ensemble_weights = {\n        'siglip': W_SIGLIP,\n        'dino': W_DINO\n    }\n    \n    try:\n        submission = robust_ensemble(FILES, ensemble_weights)\n        submission.to_csv(OUTPUT_FILE, index=False)\n        print(f\"\\nSuccess! Saved to {OUTPUT_FILE}\")\n        \n        # Stats\n        print(\"\\nStats:\")\n        print(submission['target'].describe())\n        \n    except Exception as e:\n        print(f\"\\nError during ensembling: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:52:59.302995Z","iopub.execute_input":"2026-01-19T15:52:59.303263Z","iopub.status.idle":"2026-01-19T15:52:59.362832Z","shell.execute_reply.started":"2026-01-19T15:52:59.303238Z","shell.execute_reply":"2026-01-19T15:52:59.36226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# now import timm fresh\nimport timm, sys\nprint(\"python:\", sys.version)\nprint(\"timm:\", timm.__version__)\nprint(\"timm file:\", timm.__file__)\nprint(\"dinov3 matches:\", timm.list_models(\"*dinov3*\")[:50])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:52:59.364454Z","iopub.execute_input":"2026-01-19T15:52:59.364734Z","iopub.status.idle":"2026-01-19T15:53:00.592571Z","shell.execute_reply.started":"2026-01-19T15:52:59.364715Z","shell.execute_reply":"2026-01-19T15:53:00.591876Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Config\nimport os, gc, math, cv2, numpy as np, pandas as pd\nfrom tqdm import tqdm\nimport torch, torch.nn as nn, torch.optim as optim\nfrom torch.optim.lr_scheduler import LambdaLR\nfrom torch.utils.data import Dataset, DataLoader\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\nimport time\nfrom timm.utils import ModelEmaV2\nfrom sklearn.model_selection import KFold, StratifiedGroupKFold\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ntorch.backends.cuda.matmul.allow_tf32 = True\ntorch.backends.cudnn.allow_tf32 = True\ntorch.backends.cudnn.benchmark = True\ntorch.set_float32_matmul_precision(\"high\")\n\nclass CFG:\n    CREATE_SUBMISSION = True\n    USE_TQDM        = False\n    PRETRAINED_DIR  = None\n    PRETRAINED      = True\n    BASE_PATH       = '/kaggle/input/csiro-biomass'\n    SEED            = 82947501\n    FOLDS_TO_TRAIN   = [0,1,2,3,4]\n    TRAIN_CSV       = os.path.join(BASE_PATH, 'train.csv')\n    TRAIN_IMAGE_DIR = os.path.join(BASE_PATH, 'train')\n    TEST_IMAGE_DIR = '/kaggle/input/csiro-biomass/test'\n    TEST_CSV = '/kaggle/input/csiro-biomass/test.csv'\n    SUBMISSION_DIR  = './'\n    MODEL_DIR = '/kaggle/input/baseline-dinov3/pytorch/default/1/baseline-dinov3'\n    N_FOLDS         = 5\n\n    MODEL_NAME      = 'vit_huge_plus_patch16_dinov3.lvd1689m'\n    BACKBONE_PATH   = '/kaggle/input/vit-huge-plus-patch16-dinov3-lvd1689m/vit_huge_plus_patch16_dinov3.lvd1689m_backbone.pth'\n    DINO_GRAD_CHECKPOINTING = False\n\n    IMG_SIZE        = 512\n\n    VAL_TTA_TIMES   = 1\n    TTA_STEPS       = 1\n    \n    \n    BATCH_SIZE      = 1\n    GRAD_ACC        = 4\n    NUM_WORKERS     = 4\n    EPOCHS          = 1\n    FREEZE_EPOCHS   = 0\n    WARMUP_EPOCHS   = 3\n    LR_REST         = 1e-3\n    LR_BACKBONE     = 5e-4\n    WD              = 1e-2\n    EMA_DECAY       = 0.9\n    CLIP_GRAD_NORM  = 1.0\n    PATIENCE        = 5\n    TARGET_COLS     = ['Dry_Total_g', 'GDM_g', 'Dry_Green_g']\n    DERIVED_COLS    = ['Dry_Clover_g', 'Dry_Dead_g']\n    ALL_TARGET_COLS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n    R2_WEIGHTS      = np.array([0.1, 0.1, 0.1, 0.2, 0.5])\n    LOSS_WEIGHTS    = np.array([0.1, 0.1, 0.1, 0.0, 0.0])\n    DEVICE          = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nos.makedirs(CFG.MODEL_DIR, exist_ok=True)\nprint(f'Device : {CFG.DEVICE}')\nprint(f'Backbone: {CFG.MODEL_NAME} | Input: {CFG.IMG_SIZE}')\nprint(f'Freeze Epochs: {CFG.FREEZE_EPOCHS} | Warmup: {CFG.WARMUP_EPOCHS}')\nprint(f'EMA Decay: {CFG.EMA_DECAY} | Grad Acc: {CFG.GRAD_ACC}')","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:00.59333Z","iopub.execute_input":"2026-01-19T15:53:00.593592Z","iopub.status.idle":"2026-01-19T15:53:33.521304Z","shell.execute_reply.started":"2026-01-19T15:53:00.59357Z","shell.execute_reply":"2026-01-19T15:53:33.520681Z"},"papermill":{"duration":37.523526,"end_time":"2026-01-05T05:09:15.453763","exception":false,"start_time":"2026-01-05T05:08:37.930237","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Metrics\ndef weighted_r2_score(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    r2_scores = []\n    for i in range(y_true.shape[1]):\n        yt = y_true[:, i]; yp = y_pred[:, i]\n        ss_res = np.sum((yt - yp) ** 2)\n        ss_tot = np.sum((yt - np.mean(yt)) ** 2)\n        r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n        r2_scores.append(r2)\n    r2_scores = np.array(r2_scores)\n    weighted = np.sum(r2_scores * weights) / np.sum(weights)\n    return weighted, r2_scores\n\ndef weighted_r2_score_global(y_true: np.ndarray, y_pred: np.ndarray):\n    weights = CFG.R2_WEIGHTS\n    flat_true = y_true.reshape(-1)\n    flat_pred = y_pred.reshape(-1)\n    w = np.tile(weights, y_true.shape[0])\n    mean_w = np.sum(w * flat_true) / np.sum(w)\n    ss_res = np.sum(w * (flat_true - flat_pred) ** 2)\n    ss_tot = np.sum(w * (flat_true - mean_w) ** 2)\n    global_r2 = 1 - ss_res / ss_tot if ss_tot > 0 else 0.0\n    avg_r2, per_r2 = weighted_r2_score(y_true, y_pred)\n    return global_r2, avg_r2, per_r2\n\ndef analyze_errors(val_df, y_true, y_pred, targets, top_n=5):\n    print(f'\\n--- Top {top_n} High Loss Samples per Target ---')\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    \n    for i, target in enumerate(targets):\n        errors = np.abs(y_true[:, i] - y_pred[:, i])\n        top_indices = np.argsort(errors)[::-1][:top_n]\n        \n        print(f'\\nTarget: {target}')\n        print(f'{\"Index\":<6} | {\"Image Path\":<40} | {\"True\":<10} | {\"Pred\":<10} | {\"AbsErr\":<10}')\n        print('-' * 90)\n        \n        for idx in top_indices:\n            path = val_df.iloc[idx]['image_path']\n            path_disp = os.path.basename(path)\n            t_val = y_true[idx, i]\n            p_val = y_pred[idx, i]\n            err = errors[idx]\n            print(f'{idx:<6} | {path_disp:<40} | {t_val:<10.4f} | {p_val:<10.4f} | {err:<10.4f}')\ndef analyze_errors(val_df, y_true, y_pred, targets, top_n=5):\n    print(f'\\n--- Top {top_n} High Loss Samples per Target ---')\n    y_true = np.array(y_true)\n    y_pred = np.array(y_pred)\n    \n    for i, target in enumerate(targets):\n        errors = np.abs(y_true[:, i] - y_pred[:, i])\n        top_indices = np.argsort(errors)[::-1][:top_n]\n        \n        print(f'\\nTarget: {target}')\n        header = f'{\"Index\":<6} | {\"Image Path\":<40} | {\"State\":<6} | {\"True\":<10} | {\"Pred\":<10} | {\"AbsErr\":<10}'\n        print(header)\n        print('-' * len(header))\n        \n        for idx in top_indices:\n            path = val_df.iloc[idx]['image_path']\n            path_disp = os.path.basename(path)\n            state = val_df.iloc[idx]['State'] if 'State' in val_df.columns else 'NA'\n            t_val = y_true[idx, i]\n            p_val = y_pred[idx, i]\n            err = errors[idx]\n            print(f'{idx:<6} | {path_disp:<40} | {str(state):<6} | {t_val:<10.4f} | {p_val:<10.4f} | {err:<10.4f}')\ndef compare_train_val(tr_df, val_df, targets, show_plots=True):\n    \"\"\"Quick comparison of target distributions and metadata between train and val splits.\"\"\"\n    print(\"\\n--- Train / Val Comparison ---\")\n\n    for t in targets:\n        tr = tr_df.get(t, pd.Series(dtype=float)).dropna()\n        val = val_df.get(t, pd.Series(dtype=float)).dropna()\n        print(f\"\\nTarget: {t}\")\n        print(f\"  Train: n={len(tr)} mean={tr.mean():.3f} std={tr.std():.3f} min={tr.min():.3f} max={tr.max():.3f}\")\n        print(f\"  Val  : n={len(val)} mean={val.mean():.3f} std={val.std():.3f} min={val.min():.3f} max={val.max():.3f}\")\n        if show_plots:\n            try:\n                plt.figure(figsize=(6, 3))\n                sns.kdeplot(tr, label='train', fill=True)\n                sns.kdeplot(val, label='val', fill=True)\n                plt.legend()\n                plt.title(f'Distribution: {t}')\n                plt.show()\n            except Exception as e:\n                print('  Could not plot distributions for', t, '-', e)\n\n    # Compare Sampling_Date and State if present\n    if 'Sampling_Date' in tr_df.columns:\n        try:\n            tr_dates = pd.to_datetime(tr_df['Sampling_Date'], errors='coerce')\n            val_dates = pd.to_datetime(val_df['Sampling_Date'], errors='coerce')\n            print(\"\\nSampling_Date range:\")\n            print(f\"  Train: {tr_dates.min()} -> {tr_dates.max()} (missing {tr_dates.isna().sum()})\")\n            print(f\"  Val  : {val_dates.min()} -> {val_dates.max()} (missing {val_dates.isna().sum()})\")\n        except Exception as e:\n            print('  Could not parse Sampling_Date:', e)\n    if 'State' in tr_df.columns:\n        print(\"\\nState distribution (train vs val):\")\n        tr_state = tr_df['State'].value_counts(normalize=True)\n        val_state = val_df['State'].value_counts(normalize=True)\n        state_df = pd.concat([tr_state, val_state], axis=1, keys=['train', 'val']).fillna(0)\n\n        print(state_df)\n\n    \ndef biomass_loss(outputs, labels, w=None):\n    total, gdm, green, clover, dead = outputs\n    mse = nn.MSELoss()\n    huber = nn.SmoothL1Loss(beta=5.0) # Huber loss for robust regression (beta=5.0 as recommended)\n    \n    l_green  = huber(green.squeeze(),  labels[:,0])\n    l_dead   = huber(dead.squeeze(), labels[:,1]) # Use Huber loss for Dead\n    l_clover = huber(clover.squeeze(), labels[:,2])\n    l_gdm    = huber(gdm.squeeze(),    labels[:,3])\n    l_total  = huber(total.squeeze(),  labels[:,4])\n\n    # Stack per-target losses in the SAME order as CFG.ALL_TARGET_COLS\n    losses = torch.stack([l_green, l_dead, l_clover, l_gdm, l_total])\n    # losses = torch.stack([l_green, l_dead, l_clover])\n\n    # Use provided weights, or default to CFG.R2_WEIGHTS\n    if w is None:\n        return losses.mean()\n    w = torch.as_tensor(w, device=losses.device, dtype=losses.dtype)\n    w = w / w.sum()\n    return (losses * w).sum()","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.522201Z","iopub.execute_input":"2026-01-19T15:53:33.522856Z","iopub.status.idle":"2026-01-19T15:53:33.541627Z","shell.execute_reply.started":"2026-01-19T15:53:33.52283Z","shell.execute_reply":"2026-01-19T15:53:33.541079Z"},"papermill":{"duration":0.025723,"end_time":"2026-01-05T05:09:15.502256","exception":false,"start_time":"2026-01-05T05:09:15.476533","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Transforms\ndef get_train_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.Rotate(limit=(-10, 10), p=0.3, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101),\n        A.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.05, p=0.5),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\ndef get_val_transforms():\n    return A.Compose([\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ], p=1.0)\n\ndef get_tta_transforms(mode=0):\n    # mode 0: original\n    # mode 1: hflip\n    # mode 2: vflip\n    # mode 3: rotate90\n    transforms_list = [\n        A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n    ]\n    \n    if mode == 1:\n        transforms_list.append(A.HorizontalFlip(p=1.0))\n    elif mode == 2:\n        transforms_list.append(A.VerticalFlip(p=1.0))\n    elif mode == 3:\n        transforms_list.append(A.RandomRotate90(p=1.0)) # RandomRotate90 with p=1.0 rotates 90, 180, 270 randomly? \n        # Albumentations RandomRotate90 rotates by 90, 180, 270. \n        # Reference uses transforms.RandomRotation([90, 90]) which is exactly 90 degrees.\n        # To match exactly 90 degrees in Albumentations, we might need Rotate(limit=(90,90), p=1.0)\n        # But RandomRotate90 is standard TTA. Let's use Rotate(limit=(90,90)) to be precise if that's what reference does.\n        # Reference: transforms.RandomRotation([90, 90]) -> rotates by exactly 90 degrees.\n        transforms_list.append(A.Rotate(limit=(90, 90), p=1.0, interpolation=cv2.INTER_LINEAR, border_mode=cv2.BORDER_REFLECT_101))\n\n    transforms_list.extend([\n        A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n        ToTensorV2()\n    ])\n    \n    return A.Compose(transforms_list, p=1.0)\ndef clean_image(img):\n    # 1. Safe Crop (Remove artifacts at the bottom)\n    h, w = img.shape[:2]\n    # Cut bottom 10% where artifacts often appear\n    img = img[0:int(h*0.90), :] \n\n    # 2. Inpaint Date Stamp (Remove orange text)\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    # Define orange color range (adjust as needed)\n    lower = np.array([5, 150, 150])\n    upper = np.array([25, 255, 255])\n    mask = cv2.inRange(hsv, lower, upper)\n\n    # Dilate mask to cover text edges and reduce noise\n    mask = cv2.dilate(mask, np.ones((3,3), np.uint8), iterations=2)\n\n    # Inpaint if mask is not empty\n    if np.sum(mask) > 0:\n        img = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)\n\n    return img\nclass BiomassDataset(Dataset):\n    def __init__(self, df, transform, img_dir):\n        self.df = df\n        self.transform = transform\n        self.img_dir = img_dir\n        self.paths = df['image_path'].values\n        self.labels = df[CFG.ALL_TARGET_COLS].values.astype(np.float32)\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        path = os.path.join(self.img_dir, os.path.basename(self.paths[idx]))\n        img = cv2.imread(path)\n        if img is None:\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = clean_image(img)\n        h, w, _ = img.shape\n        mid = w // 2\n        left = img[:, :mid]\n        right = img[:, mid:]\n        left = self.transform(image=left)['image']\n        right = self.transform(image=right)['image']\n        label = torch.from_numpy(self.labels[idx])\n        return left, right, label","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.542583Z","iopub.execute_input":"2026-01-19T15:53:33.542811Z","iopub.status.idle":"2026-01-19T15:53:33.570739Z","shell.execute_reply.started":"2026-01-19T15:53:33.542792Z","shell.execute_reply":"2026-01-19T15:53:33.57019Z"},"papermill":{"duration":0.021348,"end_time":"2026-01-05T05:09:15.546114","exception":false,"start_time":"2026-01-05T05:09:15.524766","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Layers\nclass LocalMambaBlock(nn.Module):\n    \"\"\"\n    Lightweight Mamba-style block (Gated CNN) from the reference notebook.\n    Efficiently mixes tokens with linear complexity.\n    \"\"\"\n    def __init__(self, dim, kernel_size=5, dropout=0.0):\n        super().__init__()\n        self.norm = nn.LayerNorm(dim)\n        # Depthwise conv mixes spatial information locally\n        self.dwconv = nn.Conv1d(dim, dim, kernel_size=kernel_size, padding=kernel_size // 2, groups=dim)\n        self.gate = nn.Linear(dim, dim)\n        self.proj = nn.Linear(dim, dim)\n        self.drop = nn.Dropout(dropout)\n\n    def forward(self, x):\n        # x: (Batch, Tokens, Dim)\n        shortcut = x\n        x = self.norm(x)\n        # Gating mechanism\n        g = torch.sigmoid(self.gate(x))\n        x = x * g\n        # Spatial mixing via 1D Conv (requires transpose)\n        x = x.transpose(1, 2)  # -> (B, D, N)\n        x = self.dwconv(x)\n        x = x.transpose(1, 2)  # -> (B, N, D)\n        # Projection\n        x = self.proj(x)\n        x = self.drop(x)\n        return shortcut + x\n","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.571515Z","iopub.execute_input":"2026-01-19T15:53:33.571787Z","iopub.status.idle":"2026-01-19T15:53:33.595326Z","shell.execute_reply.started":"2026-01-19T15:53:33.571763Z","shell.execute_reply":"2026-01-19T15:53:33.594603Z"},"papermill":{"duration":0.026162,"end_time":"2026-01-05T05:09:15.59418","exception":false,"start_time":"2026-01-05T05:09:15.568018","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Model\nclass BiomassModel(nn.Module):\n    def __init__(self, model_name, pretrained=True, backbone_path=None):\n        super().__init__()\n        self.model_name = model_name\n        self.backbone_path = backbone_path\n        \n        # 1. Load Backbone with global_pool='' to keep patch tokens\n        #    (B, 197, 1024) instead of (B, 1024)\n        self.backbone = timm.create_model(self.model_name, pretrained=False, num_classes=0, global_pool='')\n        \n        # 2. Enable Gradient Checkpointing (Crucial for ViT-Large memory!)\n        if hasattr(self.backbone, 'set_grad_checkpointing') and CFG.DINO_GRAD_CHECKPOINTING:\n            self.backbone.set_grad_checkpointing(True)\n            print(\"✓ Gradient Checkpointing enabled (saves ~50% VRAM)\")\n            \n        nf = self.backbone.num_features\n        \n        # 3. Mamba Fusion Neck\n        #    Mixes the concatenated tokens [Left, Right]\n        self.fusion = nn.Sequential(\n            LocalMambaBlock(nf, kernel_size=5, dropout=0.1),\n            LocalMambaBlock(nf, kernel_size=5, dropout=0.1)\n        )\n        \n        # 4. Pooling & Heads\n        self.pool = nn.AdaptiveAvgPool1d(1)\n        \n        # Heads (using the same logic as before, but on fused features)\n        self.head_green_raw  = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        self.head_clover_raw = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        self.head_dead_raw   = nn.Sequential(\n            nn.Linear(nf, nf//2), nn.GELU(), nn.Dropout(0.2), \n            nn.Linear(nf//2, 1), nn.Softplus()\n        )\n        \n        if pretrained:\n            self.load_pretrained()\n    \n    def load_pretrained(self):\n        try:\n            # Load weights normally\n            if self.backbone_path and os.path.exists(self.backbone_path):\n                print(f\"Loading backbone weights from local file: {self.backbone_path}\")\n                sd = torch.load(self.backbone_path, map_location='cpu')\n                # Handle common checkpoint wrappers (e.g. if saved with 'model' key)\n                if 'model' in sd: sd = sd['model']\n                elif 'state_dict' in sd: sd = sd['state_dict']\n            else:\n                # Original behavior: Download from internet\n                print(\"Downloading backbone weights...\")\n                sd = timm.create_model(self.model_name, pretrained=True, num_classes=0, global_pool='').state_dict()\n            \n            # Interpolate pos_embed if needed (for 256x256 vs 224x224)\n            if 'pos_embed' in sd and hasattr(self.backbone, 'pos_embed'):\n                pe_ck = sd['pos_embed']\n                pe_m  = self.backbone.pos_embed\n                if pe_ck.shape != pe_m.shape:\n                    print(f\"Interpolating pos_embed: {pe_ck.shape} -> {pe_m.shape}\")\n                    # (Simple interpolation logic here or rely on timm's load if strict=False handles it well enough)\n                    # For robust interpolation, use the snippet provided in previous turn\n            \n            self.backbone.load_state_dict(sd, strict=False)\n            print('Pretrained weights loaded.')\n        except Exception as e:\n            print(f'Warning: pretrained load failed: {e}')\n    \n    def forward(self, left, right):\n        # 1. Extract Tokens (B, N, D)\n        #    Note: ViT usually returns [CLS, Patch1, Patch2...]\n        #    We remove CLS token for spatial mixing, or keep it. Let's keep it.\n        x_l = self.backbone(left)\n        x_r = self.backbone(right)\n        \n        # 2. Concatenate Left and Right tokens along sequence dimension\n        #    (B, N, D) + (B, N, D) -> (B, 2N, D)\n        x_cat = torch.cat([x_l, x_r], dim=1)\n        \n        # 3. Apply Mamba Fusion\n        #    This allows tokens from Left image to interact with tokens from Right image\n        x_fused = self.fusion(x_cat)\n        \n        # 4. Global Pooling\n        #    (B, 2N, D) -> (B, D, 2N) -> (B, D, 1) -> (B, D)\n        x_pool = self.pool(x_fused.transpose(1, 2)).flatten(1)\n        \n        # 5. Prediction Heads\n        green  = self.head_green_raw(x_pool)\n        clover = self.head_clover_raw(x_pool)\n        dead   = self.head_dead_raw(x_pool)\n        \n        # Summation logic\n        gdm    = green + clover\n        total  = gdm + dead\n        \n        return total, gdm, green, clover, dead\n    \n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:53:33.596402Z","iopub.execute_input":"2026-01-19T15:53:33.596939Z","iopub.status.idle":"2026-01-19T15:53:33.619957Z","shell.execute_reply.started":"2026-01-19T15:53:33.596919Z","shell.execute_reply":"2026-01-19T15:53:33.619249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Utility Functions\ndef set_backbone_requires_grad(model: BiomassModel, requires_grad: bool):\n    for p in model.backbone.parameters():\n        p.requires_grad = requires_grad\n\n\ndef build_optimizer(model: BiomassModel):\n    # 1. Get backbone parameter IDs for exclusion\n    backbone_ids = {id(p) for p in model.backbone.parameters()}\n    \n    # 2. Separate params into backbone vs. everything else (heads, fusion, etc.)\n    backbone_params = []\n    rest_params = []\n    \n    for p in model.parameters():\n        if p.requires_grad:\n            if id(p) in backbone_ids:\n                backbone_params.append(p)\n            else:\n                rest_params.append(p)\n    \n    return optim.AdamW([\n        {'params': backbone_params, 'lr': CFG.LR_BACKBONE, 'weight_decay': CFG.WD},\n        {'params': rest_params,     'lr': CFG.LR_REST,     'weight_decay': CFG.WD},\n])\n\ndef build_scheduler(optimizer):\n    def lr_lambda(epoch):\n        e = max(0, epoch - 1)\n        if e < CFG.WARMUP_EPOCHS:\n            return float(e + 1) / float(max(1, CFG.WARMUP_EPOCHS))\n        progress = (e - CFG.WARMUP_EPOCHS) / float(max(1, CFG.EPOCHS - CFG.WARMUP_EPOCHS))\n        return 0.5 * (1.0 + math.cos(math.pi * progress))\n    return LambdaLR(optimizer, lr_lambda)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:53:33.620747Z","iopub.execute_input":"2026-01-19T15:53:33.620999Z","iopub.status.idle":"2026-01-19T15:53:33.648852Z","shell.execute_reply.started":"2026-01-19T15:53:33.620981Z","shell.execute_reply":"2026-01-19T15:53:33.648323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training and Validation Loops \n\nfrom contextlib import nullcontext\n\nUSE_BF16 = True\nAMP_DTYPE = torch.bfloat16 if USE_BF16 else torch.float16\n\nscaler = torch.amp.GradScaler(\n    'cuda',\n    enabled=(torch.cuda.is_available() and AMP_DTYPE == torch.float16)\n)\n\ndef autocast_ctx():\n    if not torch.cuda.is_available():\n        return nullcontext()\n    return torch.amp.autocast(device_type='cuda', dtype=AMP_DTYPE)\n\ndef _as_col(x: torch.Tensor, bs: int) -> torch.Tensor:\n    \"\"\"Force predictions to (B,1) regardless of model head returning (B,) or (B,1).\"\"\"\n    if x.ndim == 1:\n        return x.view(bs, 1)\n    if x.ndim == 2:\n        return x\n    return x.view(bs, 1)\n\ndef _ensure_2d_lab(lab: torch.Tensor, bs: int) -> torch.Tensor:\n    \"\"\"Force labels to (B,T).\"\"\"\n    if lab.ndim == 1:\n        return lab.view(bs, -1)\n    return lab\n\ndef _pred_pack(p_total, p_gdm, p_green, p_clover, p_dead, bs: int) -> torch.Tensor:\n    \"\"\"\n    Pack predictions in the exact order expected by your metric:\n    CFG.ALL_TARGET_COLS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n    \"\"\"\n    pg = _as_col(p_green,  bs)\n    pd = _as_col(p_dead,   bs)\n    pc = _as_col(p_clover, bs)\n    pgdm = _as_col(p_gdm,  bs)\n    pt = _as_col(p_total,  bs)\n    return torch.cat([pg, pd, pc, pgdm, pt], dim=1)  # (B,5)\n\n@torch.inference_mode()\ndef valid_epoch(eval_model, loader, device):\n    eval_model.eval()\n\n    n = len(loader.dataset)\n    n_targets = len(CFG.ALL_TARGET_COLS)\n\n    preds_cpu  = torch.empty((n, n_targets), dtype=torch.float32)\n    labels_cpu = torch.empty((n, n_targets), dtype=torch.float32)\n\n    total_loss = 0.0\n    offset = 0\n\n    for l, r, lab in loader:\n        bs = l.size(0)\n        l   = l.to(device, non_blocking=True)\n        r   = r.to(device, non_blocking=True)\n        lab = lab.to(device, non_blocking=True)\n\n        with autocast_ctx():\n            p_total, p_gdm, p_green, p_clover, p_dead = eval_model(l, r)\n            loss = biomass_loss((p_total, p_gdm, p_green, p_clover, p_dead),\n                                lab, w=CFG.LOSS_WEIGHTS)\n\n        total_loss += loss.detach().float().item() * bs\n\n        batch_pred = _pred_pack(p_total, p_gdm, p_green, p_clover, p_dead, bs).float().cpu()\n        batch_lab  = _ensure_2d_lab(lab, bs).float().cpu()\n\n        # Safety: ensure dims match (avoids silent 4/5 bugs)\n        if batch_pred.shape[1] != n_targets or batch_lab.shape[1] != n_targets:\n            raise RuntimeError(\n                f\"Target dim mismatch: pred={batch_pred.shape}, lab={batch_lab.shape}, \"\n                f\"CFG.ALL_TARGET_COLS={n_targets}\"\n            )\n\n        preds_cpu[offset:offset+bs]  = batch_pred\n        labels_cpu[offset:offset+bs] = batch_lab\n        offset += bs\n\n    pred_all    = preds_cpu.numpy()\n    true_labels = labels_cpu.numpy()\n    global_r2, avg_r2, per_r2 = weighted_r2_score_global(true_labels, pred_all)\n\n    return total_loss / n, global_r2, avg_r2, per_r2, pred_all, true_labels\n\n\n@torch.inference_mode()\ndef valid_epoch_tta(eval_model, loaders, device):\n    \"\"\"\n    Fixed: always accumulate (N,5). Works even if a head outputs (B,1).\n    Assumes each loader iterates in identical order (shuffle=False).\n    \"\"\"\n    eval_model.eval()\n    assert len(loaders) > 0\n\n    n = len(loaders[0].dataset)\n    n_targets = len(CFG.ALL_TARGET_COLS)\n\n    labels_cpu = torch.empty((n, n_targets), dtype=torch.float32)\n    preds_sum  = torch.zeros((n, n_targets), dtype=torch.float32)\n\n    total_loss = 0.0\n\n    for tta_i, loader in enumerate(loaders):\n        offset = 0\n        tta_loss = 0.0\n\n        for l, r, lab in loader:\n            bs = l.size(0)\n            l   = l.to(device, non_blocking=True)\n            r   = r.to(device, non_blocking=True)\n            lab = lab.to(device, non_blocking=True)\n\n            with autocast_ctx():\n                p_total, p_gdm, p_green, p_clover, p_dead = eval_model(l, r)\n                loss = biomass_loss((p_total, p_gdm, p_green, p_clover, p_dead),\n                                    lab, w=CFG.LOSS_WEIGHTS)\n\n            tta_loss += loss.detach().float().item() * bs\n\n            batch_pred = _pred_pack(p_total, p_gdm, p_green, p_clover, p_dead, bs).float().cpu()\n            preds_sum[offset:offset+bs] += batch_pred\n\n            if tta_i == 0:\n                batch_lab = _ensure_2d_lab(lab, bs).float().cpu()\n                if batch_lab.shape[1] != n_targets:\n                    raise RuntimeError(f\"Label dim mismatch: {batch_lab.shape} vs {n_targets}\")\n                labels_cpu[offset:offset+bs] = batch_lab\n\n            offset += bs\n\n        total_loss += tta_loss / n\n\n    avg_preds   = (preds_sum / len(loaders)).numpy()\n    true_labels = labels_cpu.numpy()\n    global_r2, avg_r2, per_r2 = weighted_r2_score_global(true_labels, avg_preds)\n\n    return total_loss / len(loaders), global_r2, avg_r2, per_r2, avg_preds, true_labels\n\n\ndef train_epoch(model, loader, opt, scheduler, device, ema: ModelEmaV2 | None = None):\n    model.train()\n    running = 0.0\n\n    opt.zero_grad(set_to_none=True)\n    itera = tqdm(loader, desc='train', leave=False) if CFG.USE_TQDM else loader\n\n    for i, (l, r, lab) in enumerate(itera):\n        bs = l.size(0)\n        l   = l.to(device, non_blocking=True)\n        r   = r.to(device, non_blocking=True)\n        lab = lab.to(device, non_blocking=True)\n\n        with autocast_ctx():\n            p_total, p_gdm, p_green, p_clover, p_dead = model(l, r)\n            loss = biomass_loss((p_total, p_gdm, p_green, p_clover, p_dead),\n                                lab, w=CFG.LOSS_WEIGHTS)\n            loss = loss / CFG.GRAD_ACC\n\n\n        if not torch.isfinite(loss):\n            print(\"Non-finite loss detected; skipping batch\")\n            opt.zero_grad(set_to_none=True)\n            continue\n\n        if scaler.is_enabled():\n            scaler.scale(loss).backward()\n        else:\n            loss.backward()\n\n        running += loss.detach().float().item() * bs * CFG.GRAD_ACC\n\n        do_step = ((i + 1) % CFG.GRAD_ACC == 0) or ((i + 1) == len(loader))\n        if do_step:\n            if scaler.is_enabled():\n                scaler.unscale_(opt)\n                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                scaler.step(opt)\n                scaler.update()\n            else:\n                torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n                opt.step()\n\n            if ema is not None:\n                ema.update(model.module if hasattr(model, \"module\") else model)\n\n            opt.zero_grad(set_to_none=True)\n\n    scheduler.step()\n    return running / len(loader.dataset)\n","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.649718Z","iopub.execute_input":"2026-01-19T15:53:33.649965Z","iopub.status.idle":"2026-01-19T15:53:33.676704Z","shell.execute_reply.started":"2026-01-19T15:53:33.649946Z","shell.execute_reply":"2026-01-19T15:53:33.676091Z"},"papermill":{"duration":0.048482,"end_time":"2026-01-05T05:09:15.664902","exception":false,"start_time":"2026-01-05T05:09:15.61642","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # === MAIN TRAINING LOOP === #\n\n# # Helper for accurate GPU timings\n# def _sync():\n#     if torch.cuda.is_available():\n#         torch.cuda.synchronize()\n\n# print('Loading data...')\n# df_long = pd.read_csv(CFG.TRAIN_CSV)\n\n# df_wide = (\n#     df_long.pivot(index='image_path', columns='target_name', values='target')\n#     .reset_index()\n# )\n# assert df_wide['image_path'].is_unique, 'Leakage risk: duplicate image_path rows'\n\n# # Merge metadata (Sampling_Date, State) for stratification\n# if 'Sampling_Date' in df_long.columns and 'State' in df_long.columns:\n#     print('Merging metadata for stratification...')\n#     meta_df = df_long[['image_path', 'Sampling_Date', 'State']].drop_duplicates()\n#     df_wide = df_wide.merge(meta_df, on='image_path', how='left')\n\n# # Keep necessary columns\n# df_wide = df_wide[['image_path', 'Sampling_Date', 'State'] + CFG.ALL_TARGET_COLS]\n# print(f'{len(df_wide)} training images')\n\n# # Use StratifiedGroupKFold\n# sgkf = StratifiedGroupKFold(n_splits=CFG.N_FOLDS, shuffle=True, random_state=CFG.SEED)\n# oof_true, oof_pred, fold_summary = [], [], []\n\n# # Split based on groups (Sampling_Date) and stratification target (State)\n# groups = df_wide['Sampling_Date']\n# y_stratify = df_wide['State']\n\n# # One place for loader kwargs (fast)\n# DL_KW = dict(\n#     num_workers=CFG.NUM_WORKERS,\n#     pin_memory=True,\n#     persistent_workers=(CFG.NUM_WORKERS > 0),\n#     prefetch_factor=4 if CFG.NUM_WORKERS > 0 else None,\n# )\n\n# for fold, (tr_idx, val_idx) in enumerate(sgkf.split(df_wide, y_stratify, groups=groups)):\n#     if fold not in CFG.FOLDS_TO_TRAIN:\n#         print(f'Skipping fold {fold} as per configuration.')\n#         continue\n\n#     print('\\n' + '='*70)\n#     print(f'FOLD {fold+1}/{CFG.N_FOLDS} | {len(tr_idx)} train / {len(val_idx)} val')\n#     print('='*70)\n\n#     # NOTE: avoid empty_cache/gc inside epoch loop; only between folds if you really want\n#     _sync()\n#     torch.cuda.empty_cache()\n#     gc.collect()\n\n#     tr_df  = df_wide.iloc[tr_idx].reset_index(drop=True)\n#     val_df = df_wide.iloc[val_idx].reset_index(drop=True)\n\n#     tr_set = BiomassDataset(tr_df, get_train_transforms(), CFG.TRAIN_IMAGE_DIR)\n\n#     # Create TTA loaders (keep TTAs as requested)\n#     val_loaders = []\n#     for mode in range(CFG.VAL_TTA_TIMES):  # 0: orig, 1: hflip, 2: vflip, 3: rot90\n#         val_set_tta = BiomassDataset(val_df, get_tta_transforms(mode), CFG.TRAIN_IMAGE_DIR)\n#         val_loader_tta = DataLoader(\n#             val_set_tta,\n#             batch_size=CFG.BATCH_SIZE,\n#             shuffle=False,\n#             drop_last=False,\n#             **{k: v for k, v in DL_KW.items() if v is not None},\n#         )\n#         val_loaders.append(val_loader_tta)\n\n#     tr_loader = DataLoader(\n#         tr_set,\n#         batch_size=CFG.BATCH_SIZE,\n#         shuffle=True,\n#         drop_last=True,\n#         **{k: v for k, v in DL_KW.items() if v is not None},\n#     )\n\n#     print('Building model...')\n#     backbone_path = getattr(CFG, 'BACKBONE_PATH', None)\n#     model = BiomassModel(CFG.MODEL_NAME, pretrained=CFG.PRETRAINED, backbone_path=backbone_path).to(CFG.DEVICE)\n\n#     # Load pretrained fold weights if available (for resuming or fine-tuning)\n#     if getattr(CFG, 'PRETRAINED_DIR', None) and os.path.isdir(CFG.PRETRAINED_DIR):\n#         pretrained_path = os.path.join(CFG.PRETRAINED_DIR, f'best_model_fold{fold}.pth')\n#         if os.path.exists(pretrained_path):\n#             try:\n#                 state = torch.load(pretrained_path, map_location='cpu')\n#                 if isinstance(state, dict) and ('model_state_dict' in state or 'state_dict' in state):\n#                     key = 'model_state_dict' if 'model_state_dict' in state else 'state_dict'\n#                     sd = state[key]\n#                 else:\n#                     sd = state\n#                 model.load_state_dict(sd, strict=False)\n#                 model.to(CFG.DEVICE)\n#                 print(f'  ✓ Loaded pretrained weights for fold {fold} from {pretrained_path}')\n#             except Exception as e:\n#                 print(f'  ✗ Failed to load pretrained fold {fold}: {e}')\n#         else:\n#             print(f'  (No pretrained file for fold {fold} at {pretrained_path})')\n#     else:\n#         print('  (No PRETRAINED_DIR configured or directory missing)')\n\n#     # Single GPU: DO NOT wrap in DataParallel\n#     # model = nn.DataParallel(model)  # <-- removed\n\n#     # Freeze/unfreeze backbone\n#     set_backbone_requires_grad(model, False)\n\n#     optimizer = build_optimizer(model)\n#     scheduler = build_scheduler(optimizer)\n\n#     # EMA on the real model\n#     ema = ModelEmaV2(model, decay=CFG.EMA_DECAY)\n\n#     best_global_r2 = -np.inf\n#     best_avg_r2 = -np.inf\n#     patience = 0\n#     best_fold_preds = None\n#     best_fold_true = None\n\n#     save_path = os.path.join(CFG.MODEL_DIR, f'best_model_fold{fold}.pth')\n\n#     for epoch in range(1, CFG.EPOCHS + 1):\n#         if epoch == CFG.FREEZE_EPOCHS + 1:\n#             patience = 0\n#             set_backbone_requires_grad(model, True)\n#             print(f'Epoch {epoch}: backbone unfrozen')\n\n#         # ---- Train timing ----\n#         _sync()\n#         t0 = time.perf_counter()\n#         tr_loss = train_epoch(model, tr_loader, optimizer, scheduler, CFG.DEVICE, ema)\n#         _sync()\n#         t1 = time.perf_counter()\n\n#         # choose eval model (EMA weights)\n#         eval_model = ema.module if ema is not None else model\n\n#         # ---- Val timing (TTA) ----\n#         _sync()\n#         t2 = time.perf_counter()\n#         val_loss, global_r2, avg_r2, per_r2, preds_fold, true_fold = valid_epoch_tta(eval_model, val_loaders, CFG.DEVICE)\n#         _sync()\n#         t3 = time.perf_counter()\n\n#         time_tr = t1 - t0\n#         time_val = t3 - t2\n#         time_ep = t3 - t0\n\n#         per_r2_str = ' | '.join([f'{CFG.ALL_TARGET_COLS[i][:5]}: {r2:.3f}' for i, r2 in enumerate(per_r2)])\n#         lrs = [pg['lr'] for pg in optimizer.param_groups]\n#         lr_str = ' '.join([f'lr{i}={lr:.3e}' for i, lr in enumerate(lrs)])\n\n#         print(\n#             f'Fold {fold} | Epoch {epoch:02d} | '\n#             f'TLoss {tr_loss:.5f} | VLoss {val_loss:.5f} | '\n#             f'avgR2 {avg_r2:.4f} | GlobalR² {global_r2:.4f} '\n#             f'{\"[BEST]\" if global_r2 > best_global_r2 else \"\"} | '\n#             f'{lr_str} | time_tr={time_tr:.1f}s time_val={time_val:.1f}s time_ep={time_ep:.1f}s'\n#         )\n#         print(f'  → {per_r2_str}')\n\n#         if global_r2 > best_global_r2:\n#             best_global_r2 = global_r2\n#             best_avg_r2 = avg_r2\n\n#             # Save EMA weights directly (no CPU clone) for speed\n#             # NOTE: safest is to save state_dict tensors as-is; this is typically fine on DGX.\n#             torch.save(eval_model.state_dict(), save_path)\n#             print(f'  → SAVED EMA weights to {save_path} (GlobalR²: {best_global_r2:.4f})')\n\n#             patience = 0\n#             best_fold_preds = preds_fold\n#             best_fold_true = true_fold\n#         else:\n#             patience += 1\n#             if patience >= CFG.PATIENCE:\n#                 print(f'  → EARLY STOP (no improvement in {CFG.PATIENCE} epochs)')\n#                 break\n\n#         # keep memory tidy but avoid heavy cache/gc churn\n#         del preds_fold, true_fold\n\n#     if best_fold_preds is not None:\n#         oof_true.append(best_fold_true)\n#         oof_pred.append(best_fold_preds)\n#         fold_summary.append({'fold': fold, 'global_r2': best_global_r2, 'avg_r2': best_avg_r2})\n\n#     # Cleanup for this fold\n#     del model, tr_loader, val_loaders, optimizer, scheduler, ema, eval_model\n#     _sync()\n#     torch.cuda.empty_cache()\n#     gc.collect()\n\n# if oof_true:\n#     oof_true_arr = np.concatenate(oof_true, axis=0)\n#     oof_pred_arr = np.concatenate(oof_pred, axis=0)\n#     oof_global_r2, oof_avg_r2, oof_per_r2 = weighted_r2_score_global(oof_true_arr, oof_pred_arr)\n\n#     print('\\nTraining complete! Models saved in:', CFG.MODEL_DIR)\n#     print('Fold summary:')\n#     for fs in fold_summary:\n#         print(f\"  Fold {fs['fold']}: Global R² = {fs['global_r2']:.4f}, Avg R² = {fs.get('avg_r2', float('nan')):.4f}\")\n#     print(f'OOF Global Weighted R²: {oof_global_r2:.4f} | OOF Avg Target R²: {oof_avg_r2:.4f}')\n#     print('OOF Per-target:', dict(zip(CFG.ALL_TARGET_COLS, [f\"{r:.4f}\" for r in oof_per_r2])))\n# else:\n#     print('No OOF predictions collected.')\n","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.67776Z","iopub.execute_input":"2026-01-19T15:53:33.678256Z","iopub.status.idle":"2026-01-19T15:53:33.702014Z","shell.execute_reply.started":"2026-01-19T15:53:33.678225Z","shell.execute_reply":"2026-01-19T15:53:33.701514Z"},"papermill":{"duration":0.016335,"end_time":"2026-01-05T05:09:15.715963","exception":false,"start_time":"2026-01-05T05:09:15.699628","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inference   \n\n# 4. DEFINE TTA TRANSFORMS\n# ===============================================================\ndef get_tta_transforms(num_transforms):\n    \"\"\"\n    Returns a list of TTA transform pipelines.\n    Each pipeline represents a different augmentation view.\n    \"\"\"\n    normalize = A.Normalize(\n        mean=[0.485, 0.456, 0.406],\n        std=[0.229, 0.224, 0.225]\n    )\n    to_tensor = ToTensorV2()\n    \n    all_tta_transforms = [\n        # View 1: Original (no flip)\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 2: Horizontal Flip\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.HorizontalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 3: Vertical Flip\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.VerticalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n        \n        # View 4: Both Flips\n        A.Compose([\n            A.Resize(CFG.IMG_SIZE, CFG.IMG_SIZE),\n            A.HorizontalFlip(p=1.0),\n            A.VerticalFlip(p=1.0),\n            normalize,\n            to_tensor\n        ]),\n    ]\n    tta_transforms = all_tta_transforms[:num_transforms]\n    return tta_transforms\n\nprint(f\"✓ TTA transforms defined ({CFG.TTA_STEPS} views)\")\n\n# ===============================================================\n# 5. CREATE TEST DATASET\n# ===============================================================\ndef clean_image(img):\n    # Safe crop (remove bottom artifacts) + inpaint orange date stamp\n    h, w = img.shape[:2]\n    img = img[0:int(h * 0.90), :]\n    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)\n    lower = np.array([5, 150, 150])\n    upper = np.array([25, 255, 255])\n    mask = cv2.inRange(hsv, lower, upper)\n    mask = cv2.dilate(mask, np.ones((3, 3), np.uint8), iterations=2)\n    if np.sum(mask) > 0:\n        img = cv2.inpaint(img, mask, 3, cv2.INPAINT_TELEA)\n    return img\n\nclass BiomassTestDataset(Dataset):\n    \"\"\"\n    Test dataset for biomass images.\n    Splits each 2000×1000 image into left and right 1000×1000 halves.\n    \"\"\"\n    def __init__(self, img_dir):\n        self.img_dir = img_dir\n        self.paths = sorted([\n            os.path.join(img_dir, f) for f in os.listdir(img_dir)\n            if f.lower().endswith(('.jpg', '.jpeg', '.png'))\n        ])\n        self.filenames = [os.path.basename(p) for p in self.paths]\n    \n    def __len__(self):\n        return len(self.paths)\n    \n    def __getitem__(self, idx):\n        path = self.paths[idx]\n        \n        # Read image\n        img = cv2.imread(path)\n        if img is None:\n            print(f\"Warning: Could not read {path}, using blank image\")\n            img = np.zeros((1000, 2000, 3), dtype=np.uint8)\n        \n        # Convert BGR to RGB\n        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n        img = clean_image(img)\n        # Split into left and right halves\n        h, w = img.shape[:2]\n        mid = w // 2\n        left = img[:, :mid].copy()\n        right = img[:, mid:].copy()\n        \n        return left, right, self.filenames[idx]\n\nprint(\"✓ Test dataset class defined\")\n\n# ===============================================================\n# 8. RUN INFERENCE WITH TTA (UPDATED to honor CFG.FOLDS_TO_TRAIN)\n# ===============================================================\n\n@torch.no_grad()\ndef predict_with_tta(model, left_np, right_np, tta_transforms):\n    \"\"\"\n    Predict using TTA with a SINGLE model.\n    \n    Args:\n        model: Single trained model\n        left_np: Left half of image (numpy array)\n        right_np: Right half of image (numpy array)\n        tta_transforms: List of augmentation transforms\n    \n    Returns:\n        numpy array: [total, gdm, green] predictions (averaged over TTA)\n    \"\"\"\n    all_tta_preds = []\n    \n    # Loop over TTA views\n    for tfm in tta_transforms:\n        # Apply transform to both halves\n        left_tensor = tfm(image=left_np)['image'].unsqueeze(0).to(CFG.DEVICE)\n        right_tensor = tfm(image=right_np)['image'].unsqueeze(0).to(CFG.DEVICE)\n        \n        total, gdm, green, clover, dead = model(left_tensor, right_tensor)\n        \n        # Extract values\n        p_total = total.cpu().item()\n        p_gdm = gdm.cpu().item()\n        p_green = green.cpu().item()\n        \n        all_tta_preds.append([p_total, p_gdm, p_green])\n    \n    # Average across TTA views\n    final_pred = np.mean(all_tta_preds, axis=0)\n    \n    return final_pred\n\n\ndef run_inference():\n    \"\"\"\n    Main inference function.\n    Returns: (predictions_array, image_filenames)\n    Notes:\n      - Now respects `CFG.FOLDS_TO_TRAIN` (if set) and averages only over successfully loaded folds.\n      - If no fold weights are found for the requested folds, an error is raised.\n    \"\"\"\n    print(\"\\n\" + \"=\"*70)\n    print(\"STARTING INFERENCE\")\n    print(\"=\"*70)\n    \n    # Create dataset and loader\n    dataset = BiomassTestDataset(CFG.TEST_IMAGE_DIR)\n    # Note: batch_size=1 is required for the current predict_with_tta implementation\n    loader = DataLoader(\n        dataset,\n        batch_size=1,  \n        shuffle=False,\n        num_workers=CFG.NUM_WORKERS,\n        pin_memory=True\n    )\n    \n    tta_transforms = get_tta_transforms(CFG.TTA_STEPS)\n    \n    # Initialize accumulator for predictions\n    # Shape: (num_samples, 3) for Total, GDM, Green\n    accumulated_preds = np.zeros((len(dataset), 3), dtype=np.float32)\n\n    # Use configured folds, fallback to full range if not set or empty\n    folds_to_use = getattr(CFG, 'FOLDS_TO_TRAIN', list(range(CFG.N_FOLDS)))\n    if not folds_to_use:\n        folds_to_use = list(range(CFG.N_FOLDS))\n\n    print(f\"Folds requested for inference: {folds_to_use}\")\n\n    # Use filenames from dataset (guaranteed consistent ordering with loader because shuffle=False)\n    filenames = dataset.filenames.copy()\n\n    successful_folds = 0\n    \n    # Loop over requested folds only\n    for fold in folds_to_use:\n        print(f\"\\nProcessing Fold {fold}...\")\n        # Load model for this fold\n        model_dir = CFG.MODEL_DIR\n        backbone_path = getattr(CFG, 'BACKBONE_PATH', None)\n        model = BiomassModel(CFG.MODEL_NAME, pretrained=False, backbone_path=backbone_path)\n        \n        # Load weights\n        weight_path = os.path.join(model_dir, f'best_model_fold{fold}.pth')\n        if not os.path.exists(weight_path):\n            print(f\"Warning: Model file {weight_path} not found! Skipping fold {fold}.\")\n            del model\n            torch.cuda.empty_cache(); gc.collect()\n            continue\n            \n        state = torch.load(weight_path, map_location='cpu')\n        # Handle state dict keys if necessary (e.g. if saved with 'model_state_dict' key)\n        if isinstance(state, dict) and ('model_state_dict' in state or 'state_dict' in state):\n             key = 'model_state_dict' if 'model_state_dict' in state else 'state_dict'\n             sd = state[key]\n        else:\n             sd = state\n        \n        model.load_state_dict(sd)\n        model.to(CFG.DEVICE)\n        model.eval()\n        \n        # Run inference for this fold\n        for i, (left, right, filename) in enumerate(tqdm(loader, desc=f\"Fold {fold}\")):\n            # left and right are batches of size 1, convert to numpy for TTA function\n            left_np = left[0].numpy()\n            right_np = right[0].numpy()\n            \n            # Predict\n            pred = predict_with_tta(model, left_np, right_np, tta_transforms)\n            accumulated_preds[i] += pred\n            \n        successful_folds += 1\n        # Cleanup model to save memory\n        del model\n        torch.cuda.empty_cache(); gc.collect()\n        \n    if successful_folds == 0:\n        raise FileNotFoundError(f\"No model weights found for requested folds: {folds_to_use}\")\n\n    # Average predictions over the number of successfully loaded folds\n    final_predictions = accumulated_preds / successful_folds\n    \n    print(f\"\\nInference complete. Successfully used {successful_folds} fold(s) out of {len(folds_to_use)} requested.\")\n    return final_predictions, filenames\n\n# ===============================================================\n# 9. POST-PROCESS PREDICTIONS\n# ===============================================================\ndef postprocess_predictions(preds_direct):\n    \"\"\"\n    Calculate derived targets from direct predictions.\n    \n    Input: (n_samples, 3) array with [total, gdm, green]\n    Output: (n_samples, 5) array with [green, dead, clover, gdm, total]\n    \"\"\"\n    print(\"\\nPost-processing predictions...\")\n    \n    # Extract direct predictions\n    pred_total = preds_direct[:, 0]\n    pred_gdm = preds_direct[:, 1]\n    pred_green = preds_direct[:, 2]\n    \n    # Calculate derived targets with non-negativity constraint\n    pred_clover = np.maximum(0, pred_gdm - pred_green)\n    pred_dead = np.maximum(0, pred_total - pred_gdm)\n    \n    # Stack in the order of ALL_TARGET_COLS\n    # ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n    preds_all = np.stack([\n        pred_green,\n        pred_dead,\n        pred_clover,\n        pred_gdm,\n        pred_total\n    ], axis=1)\n    \n    print(f\"✓ Post-processing complete\")\n    print(f\"  Output shape: {preds_all.shape}\")\n    print(f\"\\nPrediction statistics:\")\n    for i, col in enumerate(CFG.ALL_TARGET_COLS):\n        print(f\"  {col:15s}: mean={preds_all[:, i].mean():.2f}, \"\n              f\"std={preds_all[:, i].std():.2f}, \"\n              f\"min={preds_all[:, i].min():.2f}, \"\n              f\"max={preds_all[:, i].max():.2f}\")\n    \n    return preds_all\n\n\n# ===============================================================\n# 10. CREATE SUBMISSION FILE (FIXED)\n# ===============================================================\ndef create_submission(predictions, filenames):\n    \"\"\"\n    Create submission file in the required format.\n    \n    Args:\n        predictions: (n_images, 5) array with all target predictions\n        filenames: list of test image filenames\n    \"\"\"\n    print(\"\\n\" + \"=\"*70)\n    print(\"CREATING SUBMISSION FILE\")\n    print(\"=\"*70)\n    \n    # Step 0: Load test.csv first to check the image_path format\n    test_df = pd.read_csv(CFG.TEST_CSV)\n    print(f\"\\nTest CSV loaded: {len(test_df)} rows\")\n    print(f\"Sample image_path from test.csv: {test_df['image_path'].iloc[0]}\")\n    print(f\"Sample filename from predictions: {filenames[0]}\")\n    \n    # Step 1: Fix image_path format to match test.csv\n    # If test.csv has \"test/ID123.jpg\" but we have \"ID123.jpg\", add the prefix\n    test_path_example = test_df['image_path'].iloc[0]\n    if '/' in test_path_example:\n        # Extract the subdirectory prefix (e.g., \"test/\")\n        prefix = test_path_example.rsplit('/', 1)[0] + '/'\n        corrected_filenames = [prefix + fn for fn in filenames]\n        print(f\"Corrected path format: {corrected_filenames[0]}\")\n    else:\n        corrected_filenames = filenames\n    \n    # Step 2: Create wide-format DataFrame with corrected paths\n    preds_wide = pd.DataFrame(predictions, columns=CFG.ALL_TARGET_COLS)\n    preds_wide.insert(0, 'image_path', corrected_filenames)\n    \n    print(f\"\\nWide format predictions:\")\n    print(preds_wide.head())\n    \n    # Step 3: Convert to long format (melt)\n    preds_long = preds_wide.melt(\n        id_vars=['image_path'],\n        value_vars=CFG.ALL_TARGET_COLS,\n        var_name='target_name',\n        value_name='target'\n    )\n    \n    print(f\"\\nLong format predictions (first 10 rows):\")\n    print(preds_long.head(10))\n    \n    # Step 4: Debug the merge\n    print(f\"\\nDebug: Checking if paths match...\")\n    print(f\"Unique paths in test_df: {test_df['image_path'].nunique()}\")\n    print(f\"Unique paths in preds_long: {preds_long['image_path'].nunique()}\")\n    \n    common_paths = set(test_df['image_path'].unique()) & set(preds_long['image_path'].unique())\n    print(f\"Common paths found: {len(common_paths)}\")\n    \n    if len(common_paths) == 0:\n        print(\"\\n❌ ERROR: No matching paths found!\")\n        print(f\"Test CSV paths sample: {list(test_df['image_path'].unique()[:3])}\")\n        print(f\"Prediction paths sample: {list(preds_long['image_path'].unique()[:3])}\")\n        raise ValueError(\"Path mismatch between test.csv and predictions\")\n    \n    # Step 5: Merge to get sample_ids\n    submission = pd.merge(\n        test_df[['sample_id', 'image_path', 'target_name']],\n        preds_long,\n        on=['image_path', 'target_name'],\n        how='left'\n    )\n    \n    # Step 6: Keep only required columns\n    submission = submission[['sample_id', 'target']]\n    \n    # Step 7: Check for missing values\n    missing_count = submission['target'].isna().sum()\n    if missing_count > 0:\n        print(f\"\\n⚠ Warning: {missing_count} missing predictions found!\")\n        print(\"Sample missing entries:\")\n        print(submission[submission['target'].isna()].head())\n        submission.loc[submission['target'].isna(), 'target'] = 0.0\n    \n    # Step 8: Sort by sample_id\n    submission = submission.sort_values('sample_id').reset_index(drop=True)\n    \n    # Step 9: Save to CSV\n    output_path = os.path.join(CFG.SUBMISSION_DIR, 'submission70.csv')\n    submission.to_csv(output_path, index=False)\n    \n    print(f\"\\n✓ Submission file saved: {output_path}\")\n    print(f\"  Total rows: {len(submission)}\")\n    print(f\"\\nPrediction statistics:\")\n    print(f\"  Min: {submission['target'].min():.4f}\")\n    print(f\"  Max: {submission['target'].max():.4f}\")\n    print(f\"  Mean: {submission['target'].mean():.4f}\")\n    print(f\"  Non-zero values: {(submission['target'] > 0).sum()}/{len(submission)}\")\n    \n    print(f\"\\nFirst 10 rows:\")\n    print(submission.head(10))\n    print(f\"\\nLast 10 rows:\")\n    print(submission.tail(10))\n    \n    # Step 10: Validation checks\n    print(f\"\\n\" + \"=\"*70)\n    print(\"VALIDATION CHECKS\")\n    print(\"=\"*70)\n    print(f\"✓ Expected rows: {len(test_df)}\")\n    print(f\"✓ Actual rows: {len(submission)}\")\n    print(f\"✓ Match: {len(submission) == len(test_df)}\")\n    print(f\"✓ No missing values: {not submission['target'].isna().any()}\")\n    print(f\"✓ All sample_ids unique: {submission['sample_id'].is_unique}\")\n    print(f\"✓ Has non-zero predictions: {(submission['target'] > 0).any()}\")\n    \n    return submission\n\n# Create submission\n# Post-process predictions\n# Run inference\nif CFG.CREATE_SUBMISSION:\n    predictions_direct, test_filenames = run_inference()\n    predictions_all = postprocess_predictions(predictions_direct)\n    submission_df = create_submission(predictions_all, test_filenames)","metadata":{"execution":{"iopub.status.busy":"2026-01-19T15:53:33.703849Z","iopub.execute_input":"2026-01-19T15:53:33.704293Z","iopub.status.idle":"2026-01-19T15:56:47.13155Z","shell.execute_reply.started":"2026-01-19T15:53:33.704266Z","shell.execute_reply":"2026-01-19T15:56:47.130755Z"},"papermill":{"duration":0.015528,"end_time":"2026-01-05T05:09:15.754447","exception":false,"start_time":"2026-01-05T05:09:15.738919","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\n\n# =============================================================================\n# CONFIGURATION\n# =============================================================================\n# Weights for the ensemble (Must sum to 1.0)\nW_SIGLIP = 0.20\nW_DINO   = 0.80\n\nFILES = {\n    'siglip': 'submission70.csv',\n    'dino':   'submission72.csv'\n}\n\nOUTPUT_FILE = 'submission.csv'\n\n# Target definitions required for Mass Balance\nALL_TARGETS = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n\n# =============================================================================\n# HELPER FUNCTIONS\n# =============================================================================\n\ndef enforce_mass_balance(df_wide, fixed_clover=None):\n    \"\"\"\n    Applies Orthogonal Projection to enforce biological constraints:\n    1. Dry_Green_g + Dry_Clover_g = GDM_g\n    2. GDM_g + Dry_Dead_g = Dry_Total_g\n    \n    If fixed_clover is True, Dry_Clover_g is kept fixed and only other targets are adjusted.\n    \"\"\"\n    # 1. Ensure columns are in the specific order for the matrix math\n    ordered_cols = ['Dry_Green_g', 'Dry_Clover_g', 'Dry_Dead_g', 'GDM_g', 'Dry_Total_g']\n    \n    # Extract values: Shape (5, N_samples)\n    Y = df_wide[ordered_cols].values.T\n    \n    if fixed_clover:\n        # Keep Dry_Clover_g fixed, adjust only other targets\n        # We have: Green + Clover_fixed = GDM, so GDM = Green + Clover_fixed\n        # And: GDM + Dead = Total, so Total = GDM + Dead = Green + Clover_fixed + Dead\n        \n        # Note: In this specific pipeline, we also want to respect the adjusted Dry_Dead_g \n        # from the heuristics if possible, but the standard mass balance logic here \n        # recalculates GDM and Total based on components.\n        \n        clover_fixed = Y[1, :].copy()  # Dry_Clover_g is index 1\n        \n        # Adjust: GDM = Green + Clover_fixed\n        Y[3, :] = Y[0, :] + clover_fixed  # GDM_g\n        \n        # Adjust: Total = GDM + Dead\n        Y[4, :] = Y[3, :] + Y[2, :]  # Dry_Total_g\n        \n        Y_reconciled = Y\n    else:\n        # Standard projection if not fixing specific columns\n        C = np.array([\n            [1, 1, 0, -1,  0],\n            [0, 0, 1,  1, -1]\n        ])\n        C_T = C.T\n        try:\n            inv_CCt = np.linalg.inv(C @ C_T)\n            P = np.eye(5) - C_T @ inv_CCt @ C\n            Y_reconciled = P @ Y\n        except np.linalg.LinAlgError:\n            return df_wide\n\n    # 5. Transpose back to (N_samples, 5) and clip negatives\n    Y_reconciled = Y_reconciled.T\n    Y_reconciled = np.maximum(0, Y_reconciled) \n    \n    # 6. Update DataFrame\n    df_out = df_wide.copy()\n    df_out[ordered_cols] = Y_reconciled\n    \n    return df_out\n\ndef apply_heuristics(row):\n    \"\"\"\n    Applies the advanced preprocessing/post-processing rules \n    found in the competition inference notebook.\n    \"\"\"\n    target_name = row['target_name']\n    prediction = row['target']\n    \n    if target_name == \"Dry_Clover_g\":\n        # Scale down Clover\n        prediction = prediction * 0.8\n        \n    elif target_name == \"Dry_Dead_g\":\n        # Conditional scaling for Dead grass\n        if prediction > 20:\n            prediction *= 1.1\n        elif prediction < 10:\n            prediction *= 0.9\n            \n    return prediction\n\ndef robust_ensemble(file_paths, weights):\n    print(f\"--- Starting Ensemble ---\")\n    print(f\"Weights: {weights}\")\n    \n    dfs = []\n    for name, path in file_paths.items():\n        if not os.path.exists(path):\n            raise FileNotFoundError(f\"Missing file: {path}\")\n        \n        # Read and sort\n        df = pd.read_csv(path).sort_values('sample_id').reset_index(drop=True)\n        dfs.append(df)\n        print(f\"Loaded {name}: {len(df)} rows\")\n\n    # 1. Check alignment\n    base_ids = dfs[0]['sample_id']\n    if not all(df['sample_id'].equals(base_ids) for df in dfs[1:]):\n        raise ValueError(\"Sample IDs do not match between submission files!\")\n\n    # 2. Split by target_name\n    dfs_split = []\n    for df in dfs:\n        df_split = df.copy()\n        df_split[['image_id', 'target_name']] = df_split['sample_id'].str.rsplit('__', n=1, expand=True)\n        dfs_split.append(df_split)\n    \n    dino_idx = list(file_paths.keys()).index('dino')\n    siglip_idx = list(file_paths.keys()).index('siglip')\n    dino_df = dfs_split[dino_idx]\n    siglip_df = dfs_split[siglip_idx]\n    \n    # 3. Aggregation Strategy\n    ensemble_results = []\n    \n    for target in ALL_TARGETS:\n        dino_target = dino_df[dino_df['target_name'] == target].copy()\n        siglip_target = siglip_df[siglip_df['target_name'] == target].copy()\n        \n        if target == 'Dry_Clover_g':\n            # Use DINO only for Clover\n            print(f\"Using DINO-only for {target}\")\n            ensemble_results.append(dino_target[['sample_id', 'image_id', 'target_name', 'target']])\n        else:\n            # Weighted average for others\n            merged = pd.merge(\n                dino_target[['sample_id', 'image_id', 'target_name', 'target']].rename(columns={'target': 'dino_pred'}),\n                siglip_target[['sample_id', 'target']].rename(columns={'target': 'siglip_pred'}),\n                on='sample_id'\n            )\n            \n            w_vec = np.array([weights['dino'], weights['siglip']])\n            w_vec = w_vec / w_vec.sum()\n            \n            merged['target'] = merged['dino_pred'] * w_vec[0] + merged['siglip_pred'] * w_vec[1]\n            ensemble_results.append(merged[['sample_id', 'image_id', 'target_name', 'target']])\n    \n    # 4. Combine all targets\n    ensemble_df = pd.concat(ensemble_results, ignore_index=True)\n    print(\"Weighted average complete.\")\n\n    # =========================================================================\n    # NEW STEP: APPLY COMPETITION HEURISTICS\n    # =========================================================================\n    print(\"Applying Advanced Heuristics (Scaling Clover and Dead)...\")\n    # We apply this row-by-row or vectorized. \n    ensemble_df['target'] = ensemble_df.apply(apply_heuristics, axis=1)\n\n    # 5. Prepare for Mass Balance (Convert Long -> Wide)\n    wide_df = ensemble_df.pivot(index='image_id', columns='target_name', values='target').reset_index()\n\n    # 6. Apply Mass Balance\n    # Since we modified Clover and Dead using heuristics, we pass fixed_clover=True\n    # so the mass balance recalculates GDM and Total to match the new heuristics.\n    print(\"Applying Mass Balance Constraints...\")\n    wide_balanced = enforce_mass_balance(wide_df, fixed_clover=True)\n\n    # 7. Convert back (Wide -> Long)\n    long_balanced = wide_balanced.melt(\n        id_vars='image_id', \n        value_vars=ALL_TARGETS,\n        var_name='target_name',\n        value_name='target'\n    )\n\n    # Reconstruct sample_id\n    long_balanced['sample_id'] = long_balanced['image_id'] + '__' + long_balanced['target_name']\n\n    # 8. Final Formatting\n    final_submission = long_balanced[['sample_id', 'target']].sort_values('sample_id').reset_index(drop=True)\n\n    return final_submission\n\n# =============================================================================\n# EXECUTION\n# =============================================================================\nif __name__ == \"__main__\":\n    \n    ensemble_weights = {\n        'siglip': W_SIGLIP,\n        'dino': W_DINO\n    }\n    \n    try:\n        submission = robust_ensemble(FILES, ensemble_weights)\n        submission.to_csv(OUTPUT_FILE, index=False)\n        print(f\"\\nSuccess! Saved to {OUTPUT_FILE}\")\n        \n        # Stats\n        print(\"\\nStats:\")\n        print(submission['target'].describe())\n        \n    except Exception as e:\n        print(f\"\\nError during ensembling: {e}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-01-19T15:56:47.132784Z","iopub.execute_input":"2026-01-19T15:56:47.133672Z","iopub.status.idle":"2026-01-19T15:56:47.190131Z","shell.execute_reply.started":"2026-01-19T15:56:47.133627Z","shell.execute_reply":"2026-01-19T15:56:47.18954Z"}},"outputs":[],"execution_count":null}]}