{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13564124,"sourceType":"datasetVersion","datasetId":8616019},{"sourceId":13619987,"sourceType":"datasetVersion","datasetId":8655840},{"sourceId":14279367,"sourceType":"datasetVersion","datasetId":8646515},{"sourceId":14340406,"sourceType":"datasetVersion","datasetId":9154481},{"sourceId":14657818,"sourceType":"datasetVersion","datasetId":8615211}],"dockerImageVersionId":31153,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## This notebook is just a wrapper \n\n## For inference and train code, please see: https://github.com/LinkinPony/CSIRO\n\nI'm not sure if is OK to publish dinov3 weights, so I only make head weights public.\n\nFull solution will be published in a few days.","metadata":{}},{"cell_type":"code","source":"# ===== User variables (edit these three) =====\n\n\n\n# ===== Required user variables =====\nHEAD_WEIGHTS_PT_PATH = \"/kaggle/input/dinov3-regressor-weights/head/\"  # regression head-only weights (.pt)\nDINO_WEIGHTS_PT_PATH = \"/kaggle/input/dinov3-weights/\"  # frozen DINOv3 weights (.pt)\nINPUT_PATH = \"/kaggle/input/csiro-biomass\"  # either directory containing test.csv and images, or a direct path to test.csv\nOUTPUT_SUBMISSION_PATH = \"/kaggle/working/submission.csv\"\nDINOV3_PATH = \"/kaggle/input/dinov3-src\"\nPEFT_PATH = \"/kaggle/input/peft-src/src\"\nPROJECT_DIR = \"/kaggle/input/dinov3-regressor-weights/\"\n# Aggregation method for combining multiple heads' outputs (multi-head mode only)\n# Options:\n# - \"median_dry_total\": pick the single head whose Dry_Total_g is closest to the median (default)\n# - \"mean\": element-wise mean across all heads\n# - \"trimmed_mean\": drop heads with max/min Dry_Total_g, then mean remaining (fallback to mean if <3 heads)\nENSEMBLE_AGGREGATION = \"mean\"\nOUTPUT_SCALE = 1.0\n# ===== Multi-GPU model-parallel inference (Scheme B) =====\n# When running the VERY large dinov3_vit7b16 backbone on 2x16GB GPUs (e.g., Kaggle T4),\n# we split the transformer blocks across cuda:0 and cuda:1 and move the token activations\n# across devices once at the split boundary.\n#\n# Notes:\n# - This is NOT data-parallel; it aims to *split model weights* across GPUs to fit in memory.\n# - Only enabled when >= 2 CUDA devices are available and the backbone is dinov3_vit7b16.\nUSE_2GPU_MODEL_PARALLEL_FOR_VIT7B = True\n# Number of transformer blocks to place on cuda:0 (the remaining go to cuda:1).\n# For ViT7B (40 blocks), a 20/20 split is a reasonable default.\nVIT7B_MP_SPLIT_IDX = 20\n# Use fp16 weights for the backbone/head in model-parallel mode to fit on 2x16GB GPUs.\nVIT7B_MP_DTYPE = \"fp16\"  # one of: \"fp16\", \"fp32\"\n# ============================================\n# ===== Inference runtime settings (not read from YAML) =====\n# Inference batch size is intentionally decoupled from training config.\n# Default to 1 for safety (e.g., ViT7B model-parallel on 2x16GB).\nINFER_BATCH_SIZE = 1\n\n# ===== MC Dropout (head-only) =====\n# When enabled, we keep backbone in eval() and sample the regression head K times with Dropout active.\n# This provides a simple predictive uncertainty estimate and sometimes improves robustness.\nMC_DROPOUT_ENABLED = True\nMC_DROPOUT_SAMPLES = 16  # typical: 8-32\n# Optional base seed for reproducible MC sampling. Set to -1 to disable seeding.\nMC_DROPOUT_SEED = 42\n# ==========================================================\n# ===== Test-Time Augmentation (TTA) =====\n# Conservative defaults that match the repo's AugMix training policy:\n# - Horizontal flip is the only consistently used geometric augmentation.\n# - Optional multi-scale resize can be enabled, but increases compute cost.\nTTA_ENABLED = True\nTTA_HFLIP = True\nTTA_VFLIP = True\n# Scales are multipliers applied to cfg.data.image_size (H,W) and will be rounded to multiples of 16.\n# Typical options: [1.0] (off), or [0.9, 1.0, 1.1] (heavier).\nTTA_SCALES = [1.0]\n# =======================================\n# ===== TabPFN inference (optional) =====\n# When enabled, this script will:\n#   1) Load CSIRO `train.csv`, pivot to image-level rows\n#   2) Extract **image-model head penultimate features** (pre-final-linear) for train + test images\n#   3) Train TabPFN on the FULL train.csv (no CV)\n#   4) Predict all test images and write `submission.csv`\n#\n# IMPORTANT:\n# - TabPFN weights are loaded **LOCAL-ONLY** from TABPFN_WEIGHTS_CKPT_PATH.\n# - No HuggingFace download / auth is attempted in this script.\nTABPFN_ENABLED = False\n\n# Optional: path to TabPFN python package source (offline-friendly).\n# If you have tabpfn installed in the environment, you can ignore this.\nTABPFN_PATH = \"/kaggle/input/dinov3-regressor-weights/third_party/TabPFN/src\"\n\n# TabPFN weights path can be EITHER:\n#  - a single checkpoint file (.ckpt), OR\n#  - a directory containing one or more .ckpt files (the script will auto-select one).\n#\n# REQUIRED when TABPFN_ENABLED=True: local TabPFN 2.5 regressor checkpoint (.ckpt)\nTABPFN_WEIGHTS_CKPT_PATH = \"/kaggle/input/tabfpn/tabpfn-v2.5-regressor-v2.5_low-skew.ckpt\"\n\n# TabPFN runtime params\nTABPFN_DEVICE = \"auto\"  # \"auto\"|\"cpu\"|\"cuda\"|\"cuda:0\"|...\nTABPFN_N_ESTIMATORS = 128\nTABPFN_FIT_MODE = \"fit_preprocessors\"\nTABPFN_INFERENCE_PRECISION = \"auto\"\nTABPFN_IGNORE_PRETRAINING_LIMITS = True\nTABPFN_N_JOBS = 1  # MultiOutputRegressor parallelism (each output fits its own TabPFNRegressor)\nTABPFN_ENABLE_TELEMETRY = False\nTABPFN_MODEL_CACHE_DIR = \"\"  # optional; e.g. \"/tmp/tabpfn_cache\"\n\n# Training data for TabPFN:\n# - Default: automatically use <dataset_root>/train.csv, where dataset_root is resolved from INPUT_PATH.\n# - Override: set to an explicit path to train.csv.\nTABPFN_TRAIN_CSV_PATH = \"/kaggle/input/csiro-biomass/train.csv\"  # optional override\n\n# Penultimate feature extraction settings\nTABPFN_FEATURE_FUSION = \"mean\"  # \"mean\" | \"concat\" (concat can exceed TabPFN feature limits)\nTABPFN_FEATURE_BATCH_SIZE = 8\nTABPFN_FEATURE_NUM_WORKERS = 8\nTABPFN_FEATURE_CACHE_PATH_TRAIN = \"\"  # optional .pt cache\nTABPFN_FEATURE_CACHE_PATH_TEST = \"\"  # optional .pt cache\n# Penultimate feature extraction settings\n# Feature mode (matches configs/train_tabpfn.yaml):\n# - \"head_penultimate\": use regression head penultimate (pre-final-linear) features (default; existing behavior)\n# - \"dinov3_only\"     : use frozen DINOv3 CLS token features (no LoRA, no head)\nTABPFN_FEATURE_MODE = \"head_penultimate\"\n# TABPFN_FEATURE_MODE = \"dinov3_only\"\n\n# TabPFN output constraint (postprocess):\n# When enabled, enforce a strict coupling between Dry_Total_g and (Clover/Dead/Green) via:\n#   total_sum = clover + dead + green\n#   total_final = (total_pred + total_sum) / 2\n#   (clover, dead, green) := total_final * (component / total_sum)  (uniform when total_sum≈0)\n#   gdm := clover + green\nTABPFN_RATIO_STRICT = False\n# ======================================================================\n# ===== Transductive affine calibration (post-hoc; no backprop) =====\n# When enabled, we match the test prediction distribution to the training distribution\n# in an evaluation-aligned space while preserving hard constraints:\n#   Total = Clover + Dead + Green\n#   GDM   = Clover + Green\n#\n# Implementation detail (in src/inference/pipeline.py):\n# - Calibrate Total in log1p(grams)\n# - Calibrate composition in logits space over [Clover, Dead, Green]\nTRANSDUCTIVE_CALIBRATION_ENABLED = False\n# Strength of the calibration in [0,1]. 0 -> no-op, 1 -> full mean/std match.\nTRANSDUCTIVE_CALIBRATION_LAM = 1\n# Robust quantile clip for estimating mean/std from distributions (lo, hi).\nTRANSDUCTIVE_CALIBRATION_Q_CLIP = (0.01, 0.99)\n# Clamp scale factors for stability.\nTRANSDUCTIVE_CALIBRATION_A_CLIP_TOTAL = (0.0, 10000)\nTRANSDUCTIVE_CALIBRATION_A_CLIP_RATIO = (0.0, 10000)\n# If False, only calibrate Total (keep original composition).\nTRANSDUCTIVE_CALIBRATION_CALIBRATE_RATIO = True\n# ================================================================","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n\n\nimport os\nimport sys\n\n\ndef _add_import_root(path: str, *, package_name: str | None = None) -> None:\n    \"\"\"\n    Add a directory to sys.path such that `import <package_name>` works.\n\n    - If `path` is a *repo root* containing `<package_name>/__init__.py`, we add `path`.\n    - If `path` is the *package directory* itself (basename == package_name), we add its parent.\n\n    This avoids accidental shadowing of stdlib modules (e.g., adding dinov3's package dir\n    directly would expose `logging/` as a top-level package and break `import logging`).\n    \"\"\"\n    p = os.path.abspath(path) if path else \"\"\n    if not (p and os.path.isdir(p)):\n        return\n\n    root = p\n    if package_name:\n        pkg_init = os.path.join(p, package_name, \"__init__.py\")\n        if os.path.isfile(pkg_init):\n            root = p\n        else:\n            # If user passed the package dir itself, use its parent.\n            if os.path.basename(p) == package_name and os.path.isfile(os.path.join(p, \"__init__.py\")):\n                root = os.path.dirname(p)\n\n    if root and os.path.isdir(root) and root not in sys.path:\n        # Put vendor roots first so we prefer bundled dependencies over system installs.\n        sys.path.insert(0, root)\n\n\ndef main() -> None:\n    # Prefer local dinov3 / peft bundles when present (offline-friendly).\n    _add_import_root(DINOV3_PATH, package_name=\"dinov3\")\n    _add_import_root(PEFT_PATH, package_name=\"peft\")\n\n    # Validate project directory and import project modules.\n    project_dir_abs = os.path.abspath(PROJECT_DIR) if PROJECT_DIR else \"\"\n    configs_dir = os.path.join(project_dir_abs, \"configs\")\n    src_dir = os.path.join(project_dir_abs, \"src\")\n    if not (project_dir_abs and os.path.isdir(project_dir_abs)):\n        raise RuntimeError(\"PROJECT_DIR must point to the repository root containing `configs/` and `src/`.\")\n    if not os.path.isdir(configs_dir):\n        raise RuntimeError(f\"configs/ not found under PROJECT_DIR: {configs_dir}\")\n    if not os.path.isdir(src_dir):\n        raise RuntimeError(f\"src/ not found under PROJECT_DIR: {src_dir}\")\n\n    def _resolve_under_project_dir(p: str) -> str:\n        \"\"\"\n        Resolve a possibly-relative path.\n\n        Kaggle notebooks typically run from `/kaggle/working`, while the repo lives under\n        `/kaggle/input/<dataset>/...`. Users often set PROJECT_DIR correctly but keep\n        vendor paths like `third_party/dinov3` relative. This helper makes those paths\n        behave as-if they were relative to PROJECT_DIR.\n        \"\"\"\n        raw = str(p or \"\").strip()\n        if not raw:\n            return raw\n        if os.path.isabs(raw) and os.path.exists(raw):\n            return raw\n        # First: resolve relative to current working dir (backward-compatible)\n        cwd_abs = os.path.abspath(raw)\n        if os.path.exists(cwd_abs):\n            return cwd_abs\n        # Fallback: resolve relative to PROJECT_DIR\n        cand = os.path.join(project_dir_abs, raw)\n        if os.path.exists(cand):\n            return os.path.abspath(cand)\n        return raw\n\n    dinov3_path_resolved = _resolve_under_project_dir(str(DINOV3_PATH))\n    peft_path_resolved = _resolve_under_project_dir(str(PEFT_PATH))\n    # Re-add vendor roots using PROJECT_DIR-relative resolution (helps on Kaggle).\n    _add_import_root(dinov3_path_resolved, package_name=\"dinov3\")\n    _add_import_root(peft_path_resolved, package_name=\"peft\")\n    _add_import_root(project_dir_abs)\n\n    from src.inference.settings import InferenceSettings  # noqa: E402\n\n    settings = InferenceSettings(\n        head_weights_pt_path=str(HEAD_WEIGHTS_PT_PATH),\n        dino_weights_pt_path=str(DINO_WEIGHTS_PT_PATH),\n        input_path=str(INPUT_PATH),\n        output_submission_path=str(OUTPUT_SUBMISSION_PATH),\n        project_dir=str(PROJECT_DIR),\n        dinov3_path=str(dinov3_path_resolved),\n        peft_path=str(peft_path_resolved),\n        use_2gpu_model_parallel_for_vit7b=bool(USE_2GPU_MODEL_PARALLEL_FOR_VIT7B),\n        vit7b_mp_split_idx=int(VIT7B_MP_SPLIT_IDX),\n        vit7b_mp_dtype=str(VIT7B_MP_DTYPE),\n        infer_batch_size=int(INFER_BATCH_SIZE),\n        tta_enabled=bool(TTA_ENABLED),\n        tta_hflip=bool(TTA_HFLIP),\n        tta_vflip=bool(TTA_VFLIP),\n        tta_scales=tuple(float(x) for x in (TTA_SCALES or [1.0])),\n        mc_dropout_enabled=bool(MC_DROPOUT_ENABLED),\n        mc_dropout_samples=int(MC_DROPOUT_SAMPLES),\n        mc_dropout_seed=int(MC_DROPOUT_SEED),\n        transductive_calibration_enabled=bool(TRANSDUCTIVE_CALIBRATION_ENABLED),\n        transductive_calibration_lam=float(TRANSDUCTIVE_CALIBRATION_LAM),\n        transductive_calibration_q_clip=tuple(float(x) for x in TRANSDUCTIVE_CALIBRATION_Q_CLIP),\n        transductive_calibration_a_clip_total=tuple(float(x) for x in TRANSDUCTIVE_CALIBRATION_A_CLIP_TOTAL),\n        transductive_calibration_a_clip_ratio=tuple(float(x) for x in TRANSDUCTIVE_CALIBRATION_A_CLIP_RATIO),\n        transductive_calibration_calibrate_ratio=bool(TRANSDUCTIVE_CALIBRATION_CALIBRATE_RATIO),\n    )\n\n    if bool(TABPFN_ENABLED):\n        from dataclasses import replace  # noqa: E402\n\n        from src.inference.tabpfn import TabPFNSubmissionSettings, run_tabpfn_submission  # noqa: E402\n\n        # All TabPFN params are loaded from configs/train_tabpfn.yaml.\n        from src.inference.tabpfn import load_tabpfn_submission_settings_from_yaml  # noqa: E402\n\n        tabpfn_settings = load_tabpfn_submission_settings_from_yaml(project_dir=str(project_dir_abs))\n        # Optional user overrides (resolved relative to PROJECT_DIR when possible).\n        tabpfn_path_resolved = _resolve_under_project_dir(str(TABPFN_PATH))\n        tabpfn_ckpt_resolved = _resolve_under_project_dir(str(TABPFN_WEIGHTS_CKPT_PATH))\n        tabpfn_train_csv_resolved = _resolve_under_project_dir(str(TABPFN_TRAIN_CSV_PATH))\n        if isinstance(tabpfn_path_resolved, str) and tabpfn_path_resolved.strip():\n            tabpfn_settings = replace(tabpfn_settings, tabpfn_python_path=str(tabpfn_path_resolved))\n        if isinstance(tabpfn_ckpt_resolved, str) and tabpfn_ckpt_resolved.strip():\n            tabpfn_settings = replace(tabpfn_settings, weights_ckpt_path=str(tabpfn_ckpt_resolved))\n        if isinstance(tabpfn_train_csv_resolved, str) and tabpfn_train_csv_resolved.strip():\n            tabpfn_settings = replace(tabpfn_settings, train_csv_path=str(tabpfn_train_csv_resolved))\n\n        run_tabpfn_submission(settings=settings, tabpfn=tabpfn_settings)\n        return\n\n    from src.inference.pipeline import run  # noqa: E402\n\n    run(settings)\n\n\nif __name__ == \"__main__\":\n    main()\n\n\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}