{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"}],"dockerImageVersionId":31154,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:41:13.252939Z","iopub.execute_input":"2025-11-01T22:41:13.253142Z","iopub.status.idle":"2025-11-01T22:41:14.698483Z","shell.execute_reply.started":"2025-11-01T22:41:13.253124Z","shell.execute_reply":"2025-11-01T22:41:14.697505Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 0. Setup\n\n- Installs (if missing): `timm`, `albumentations`, `pytorch-lightning`\n- Imports and global config\n","metadata":{}},{"cell_type":"code","source":"import os\nif not os.environ.get(\"KAGGLE_URL_BASE\"):\n    # only when running locally, not on Kaggle scoring\n    !pip install --quiet \"transformers==4.37.2\"\n    !pip install timm albumentations pytorch-lightning \n    !pip install -q \"pytorch-lightning==2.3.3\" \"lightning-utilities==0.11.6\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:41:14.699893Z","iopub.execute_input":"2025-11-01T22:41:14.700272Z","iopub.status.idle":"2025-11-01T22:41:14.706166Z","shell.execute_reply.started":"2025-11-01T22:41:14.700251Z","shell.execute_reply":"2025-11-01T22:41:14.705201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os, gc, json, time, random, math, glob, warnings\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nimport cv2\nimport albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport timm\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import seed_everything\nfrom pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor, EarlyStopping\nfrom pytorch_lightning.loggers import CSVLogger\n\nwarnings.filterwarnings('ignore')\n\nimport os\nos.environ[\"HF_HUB_OFFLINE\"] = \"1\"           # timm will not try to hit the hub\nos.environ[\"HF_HUB_DISABLE_TELEMETRY\"] = \"1\"\n\nCFG = {\n    'seed': 42,\n    'img_size': 512,          # Use 512/576 lateron\n    'batch_size': 16,\n    'num_workers': 2,\n    'epochs': 20,             # increase 20-25 if time allows\n    'lr': 2e-4,\n    'weight_decay': 1e-5,\n    'backbone': 'tf_efficientnetv2_s_in21k',  # good speed/accuracy trade-off\n    'train_val_split': 0.15,\n    'tta': 4,                 # set 0 to disable TTA\n    'precision': '16-mixed',  # use AMP on Kaggle GPU\n}\n\nCOMPETITION = \"csiro-biomass\"\nDATA_DIR = Path(\"/kaggle/input/csiro-biomass\")\nOUTPUT_DIR = Path(\"./outputs\"); OUTPUT_DIR.mkdir(parents=True, exist_ok=True)\n\nseed_everything(CFG['seed'], workers=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:41:14.707125Z","iopub.execute_input":"2025-11-01T22:41:14.707392Z","iopub.status.idle":"2025-11-01T22:42:05.552434Z","shell.execute_reply.started":"2025-11-01T22:41:14.707368Z","shell.execute_reply":"2025-11-01T22:42:05.551784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_csv= DATA_DIR/\"sample_submission.csv\"\n\nsample = pd.read_csv(sample_csv)\nsample.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:05.553253Z","iopub.execute_input":"2025-11-01T22:42:05.553780Z","iopub.status.idle":"2025-11-01T22:42:05.584133Z","shell.execute_reply.started":"2025-11-01T22:42:05.553749Z","shell.execute_reply":"2025-11-01T22:42:05.583446Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 1. Load data\n\nThe competition provides:\n- **`train.csv`** with image IDs, metadata (species, location, etc.), and 5 target columns\n- **`test.csv`** with image IDs and metadata (no targets)\n- **`images/`** folder with JPG/PNG files","metadata":{}},{"cell_type":"code","source":"\ntrain_csv = DATA_DIR/'train.csv'\ntest_csv  = DATA_DIR/'test.csv'\n\ntrain = pd.read_csv(train_csv)\ntest  = pd.read_csv(test_csv)\n# --- Image roots (robust to different layouts) ---\ndef existing(*paths):\n    for p in paths:\n        if p.exists(): return p\n    return None\n\n# If CSV already has absolute paths, prefer using them directly\nUSE_ABS = (\"image_path\" in train.columns) or (\"image_path\" in test.columns)\n\nTRAIN_IMG_DIR = existing(\n    DATA_DIR/'train/images',\n    DATA_DIR/'train',\n    DATA_DIR/'images',\n    DATA_DIR\n)\nTEST_IMG_DIR = existing(\n    DATA_DIR/'test/images',\n    DATA_DIR/'test',\n    DATA_DIR/'images',\n    DATA_DIR\n)\n\nif not USE_ABS:\n    assert TRAIN_IMG_DIR is not None, \"Could not locate train image directory.\"\n    assert TEST_IMG_DIR  is not None, \"Could not locate test image directory.\"\nprint(\"USE_ABS:\", USE_ABS)\nprint(\"TRAIN_IMG_DIR:\", TRAIN_IMG_DIR)\nprint(\"TEST_IMG_DIR:\", TEST_IMG_DIR)\n\nprint(train.shape, test.shape)\ndisplay(train.head(3))\ndisplay(test.head(3))\n\nIMG_DIR = DATA_DIR/'train'\nassert IMG_DIR.exists(), f\"Images dir not found: {IMG_DIR}\"\n# detect schema & normalize to WIDE for training \n\n# ID column\ncands_id = [c for c in train.columns if c.lower() in [\"image_id\",\"image\",\"id\",\"sample_id\"] or \"image\" in c.lower()]\nIDCOL = cands_id[0] if cands_id else train.columns[0]\n\ndef to_image_id(x):\n    x = str(x)\n    if \"/\" in x or \"\\\\\" in x:\n        base = x.split(\"/\")[-1].split(\"\\\\\")[-1]\n        return base.rsplit(\".\", 1)[0]\n    return x.rsplit(\".\", 1)[0]\n\nif \"image_path\" in train.columns and IDCOL != \"image_id\":\n    train[\"image_id\"] = train[\"image_path\"].apply(to_image_id)\n    IDCOL = \"image_id\"\nif \"image_path\" in test.columns and \"image_id\" not in test.columns:\n    test[\"image_id\"] = test[\"image_path\"].apply(to_image_id)\n\n# target list\nTARGETS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\n\n# Decide name/value columns for LONG schema\nname_col = 'target_name' if 'target_name' in train.columns else ('target' if train['target'].dtype=='O' else None)\nvalue_col = None\nfor cand in ['value','target','target_value','biomass','biomass_g']:\n    if cand in train.columns and np.issubdtype(train[cand].dtype, np.number):\n        value_col = cand\n        break\n\nhas_wide_targets = all(t in train.columns for t in TARGETS)\nis_long_like = (name_col is not None) and (value_col is not None)\n\nif is_long_like and not has_wide_targets:\n    \n    train[name_col] = train[name_col].astype(str).str.strip()\n\n    # pivot -> wide (may include more than the 5 official names)\n    wide = (train\n            .pivot_table(index=IDCOL, columns=name_col, values=value_col, aggfunc='mean')\n            .reset_index())\n\n    # bring back a few metadata columns \n    meta_cols = ['Sampling_Date','State','Species','Pre_GSHH_NDVI','Height_Ave_cm']\n    meta_cols = [c for c in meta_cols if c in train.columns]\n    if meta_cols:\n        meta_first = train.drop_duplicates(subset=[IDCOL])[ [IDCOL] + meta_cols ]\n        wide = wide.merge(meta_first, on=IDCOL, how='left')\n\n    # keep only the 5 official targets if others exist\n    present_targets = [t for t in TARGETS if t in wide.columns]\n    if len(present_targets) < len(TARGETS):\n        print(\"Some targets missing in train after pivot:\",\n              [t for t in TARGETS if t not in wide.columns])\n        # continue with intersection to let EDA/train run\n    train = wide\n\n# Final check\npresent_targets = [t for t in TARGETS if t in train.columns]\nif not present_targets:\n    raise ValueError(f\"No official targets found after normalization. \"\n                     f\"Columns present: {list(train.columns)}\")\n\nTARGETS = present_targets\n\ndisplay(train.head())\nprint(\"IDCOL:\", IDCOL)\nprint(\"Detected schema:\", \"long→wide (target_name/target)\" if is_long_like and not has_wide_targets else \"wide\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:05.585943Z","iopub.execute_input":"2025-11-01T22:42:05.586180Z","iopub.status.idle":"2025-11-01T22:42:05.676326Z","shell.execute_reply.started":"2025-11-01T22:42:05.586163Z","shell.execute_reply":"2025-11-01T22:42:05.675561Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 2. EDA (compact, visual)\n\nWe'll quickly check:\n- Target distributions (raw + log1p)\n- Pairwise correlations of targets\n- Metadata frequencies (species, season, site if provided)\n- Image statistics (shape, brightness, contrast) sampled\n- Visual grids by **`Dry_Total_g`** quantile bins\n\n\n","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\ndef plot_hist(series, title, bins=50):\n    plt.figure(figsize=(6,4))\n    plt.hist(series.values, bins=bins)\n    plt.title(title); plt.xlabel('value'); plt.ylabel('count')\n    plt.show()\n\n# Target distributions\nfor t in TARGETS:\n    plot_hist(train[t], f\"{t} (raw)\")\n    plot_hist(np.log1p(train[t]), f\"log1p({t})\")\n\n# Correlation\nplt.figure(figsize=(5,4))\ncorr = train[TARGETS].corr()\nim = plt.imshow(corr, cmap='viridis')\nplt.xticks(range(len(TARGETS)), TARGETS, rotation=45, ha='right')\nplt.yticks(range(len(TARGETS)), TARGETS)\nplt.colorbar(im, fraction=0.046, pad=0.04)\nplt.title(\"Target Correlation\")\nplt.tight_layout(); plt.show()\n\n# Metadata quick looks\nmeta_cols = [c for c in train.columns if c not in TARGETS+[IDCOL]]\nfor c in meta_cols[:6]:  # show a few\n    if train[c].dtype == 'object' or train[c].nunique()<50:\n        vc = train[c].value_counts().head(20)\n        plt.figure(figsize=(6,3))\n        vc.plot(kind='bar')\n        plt.title(f\"{c} (top 20)\")\n        plt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:05.677232Z","iopub.execute_input":"2025-11-01T22:42:05.677591Z","iopub.status.idle":"2025-11-01T22:42:08.612724Z","shell.execute_reply.started":"2025-11-01T22:42:05.677570Z","shell.execute_reply":"2025-11-01T22:42:08.611774Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Image shapes and brightness sample\nsample_ids = train[IDCOL].sample(min(200, len(train)), random_state=CFG['seed']).tolist()\nshapes, means, stds = [], [], []\nfor img_id in tqdm(sample_ids, desc=\"Image stats\"):\n    # check all extension\n    for ext in (\".jpg\",\".jpeg\",\".png\",\".bmp\"):\n        p = IMG_DIR/f\"{img_id}{ext}\"\n        if p.exists():\n            img = cv2.imread(str(p), cv2.IMREAD_COLOR)\n            if img is None: continue\n            shapes.append(img.shape[:2])\n            means.append(img.mean())\n            stds.append(img.std())\n            break\n\nplt.figure(figsize=(6,4))\nplt.hist(means, bins=50); plt.title(\"Brightness (mean pixel)\"); plt.show()\nplt.figure(figsize=(6,4))\nplt.hist(stds, bins=50); plt.title(\"Contrast proxy (pixel std)\"); plt.show()\n\n# Image grid by Dry_Total_g quantiles\nqt = pd.qcut(train['Dry_Total_g'], 4, labels=False, duplicates='drop')\ntrain['_qt'] = qt\n\ndef show_grid(df, title, rows=2, cols=4):\n    plt.figure(figsize=(cols*3, rows*3))\n    subset = df.sample(min(rows*cols, len(df)), random_state=CFG['seed'])\n    for i,(idx,row) in enumerate(subset.iterrows()):\n        img_id = row[IDCOL]\n        path = None\n        for ext in ('.jpg','.jpeg','.png','.bmp'):\n            p = IMG_DIR/f\"{img_id}{ext}\"\n            if p.exists():\n                path = p; break\n        if path is None: continue\n        img = cv2.cvtColor(cv2.imread(str(path)), cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (256,256))\n        plt.subplot(rows,cols,i+1)\n        plt.imshow(img); plt.axis('off')\n        tt = f\"{row['Dry_Total_g']:.0f}g\"\n        plt.title(tt, fontsize=9)\n    plt.suptitle(title)\n    plt.tight_layout(); plt.show()\n\nfor q in sorted(train['_qt'].dropna().unique()):\n    show_grid(train[train['_qt']==q], f\"Samples – Dry_Total_g quantile {int(q)}\")\n\ntrain.drop(columns=['_qt'], errors='ignore', inplace=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:08.613897Z","iopub.execute_input":"2025-11-01T22:42:08.614207Z","iopub.status.idle":"2025-11-01T22:42:34.847040Z","shell.execute_reply.started":"2025-11-01T22:42:08.614185Z","shell.execute_reply":"2025-11-01T22:42:34.846159Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Correlation","metadata":{}},{"cell_type":"code","source":"import numpy as np, pandas as pd, matplotlib.pyplot as plt, seaborn as sns\n\n# columns that may exist\nTARGETS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\nMETA    = [c for c in ['Pre_GSHH_NDVI','Height_Ave_cm'] if c in train.columns]\nNUMCOLS = [c for c in TARGETS+META if c in train.columns]\n\nassert len(NUMCOLS) >= 2, f\"Need at least 2 numeric columns; found {NUMCOLS}\"\n\n# Spearman is robust to skew\ncorr_s = train[NUMCOLS].corr(method='spearman')\ncorr_p = train[NUMCOLS].corr(method='pearson')\n\nfig, axes = plt.subplots(1, 2, figsize=(12, 4))\nsns.heatmap(corr_s, vmin=-1, vmax=1, cmap=\"RdBu_r\", annot=True, fmt=\".2f\", ax=axes[0])\naxes[0].set_title(\"Spearman correlation\")\n\nsns.heatmap(corr_p, vmin=-1, vmax=1, cmap=\"RdBu_r\", annot=True, fmt=\".2f\", ax=axes[1])\naxes[1].set_title(\"Pearson correlation\")\nplt.tight_layout(); plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:34.848405Z","iopub.execute_input":"2025-11-01T22:42:34.849201Z","iopub.status.idle":"2025-11-01T22:42:35.819679Z","shell.execute_reply.started":"2025-11-01T22:42:34.849156Z","shell.execute_reply":"2025-11-01T22:42:35.818765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Histogram","metadata":{}},{"cell_type":"code","source":"cols = [c for c in TARGETS if c in train.columns]\nassert len(cols) > 0, \"No biomass target columns found.\"\n\nrows = 1\ncols_per_row = len(cols)\nfig, axes = plt.subplots(rows, cols_per_row, figsize=(4*cols_per_row, 3), squeeze=False)\n\nfor j, tgt in enumerate(cols):\n    ax = axes[0, j]\n    x = train[tgt].dropna()\n    ax.hist(x, bins=40, alpha=0.65, label=\"raw\")\n    ax2 = ax.twinx()\n    ax2.hist(np.log1p(x), bins=40, alpha=0.35, color='tab:orange', label=\"log1p\")\n    ax.set_title(tgt)\n    ax.set_xlabel(\"value\"); ax.set_ylabel(\"count\")\n    ax2.set_ylabel(\"count (log1p)\")\nfig.suptitle(\"Biomass distributions: raw vs log1p (orange)\", y=1.03)\nplt.tight_layout(); plt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:35.820547Z","iopub.execute_input":"2025-11-01T22:42:35.821001Z","iopub.status.idle":"2025-11-01T22:42:37.368862Z","shell.execute_reply.started":"2025-11-01T22:42:35.820982Z","shell.execute_reply":"2025-11-01T22:42:37.367897Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Scatter plot","metadata":{}},{"cell_type":"code","source":"if 'Pre_GSHH_NDVI' in train.columns and 'Dry_Total_g' in train.columns:\n    df = train[['Pre_GSHH_NDVI','Dry_Total_g','State']].dropna()\n    # Density view to handle many points\n    plt.figure(figsize=(6,4))\n    plt.hexbin(df['Pre_GSHH_NDVI'], df['Dry_Total_g'], gridsize=40, cmap='viridis')\n    plt.colorbar(label='count')\n    sns.regplot(x='Pre_GSHH_NDVI', y='Dry_Total_g', data=df, scatter=False, color='red', ci=None)\n    plt.title('NDVI vs Dry_Total_g (with linear trend)')\n    plt.xlabel('Pre_GSHH_NDVI'); plt.ylabel('Dry_Total_g (g)')\n    plt.tight_layout(); plt.show()\n\n    # Optional: color by State (if many states, this can be busy)\n    if 'State' in train.columns and train['State'].nunique() <= 8:\n        plt.figure(figsize=(7,4))\n        sns.scatterplot(data=df, x='Pre_GSHH_NDVI', y='Dry_Total_g', hue='State', s=18, alpha=0.7)\n        plt.title('NDVI vs Dry_Total_g by State')\n        plt.tight_layout(); plt.show()\nelse:\n    print(\"Need both 'Pre_GSHH_NDVI' and 'Dry_Total_g' to plot the scatter.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:37.369817Z","iopub.execute_input":"2025-11-01T22:42:37.370412Z","iopub.status.idle":"2025-11-01T22:42:38.024419Z","shell.execute_reply.started":"2025-11-01T22:42:37.370384Z","shell.execute_reply":"2025-11-01T22:42:38.023537Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Setup & helpers for deeper EDA","metadata":{}},{"cell_type":"code","source":"# --- Paths & columns ---\nfrom pathlib import Path\nimport pandas as pd, numpy as np, cv2, matplotlib.pyplot as plt\n\nCOMP = \"csiro-biomass\"\nDATA_ROOT = Path(f\"/kaggle/input/{COMP}\")\nTRAIN_DIR = (DATA_ROOT / \"train\")  # images\nassert TRAIN_DIR.exists(), TRAIN_DIR\n\nIDCOL   = \"image_id\"\nTARGET = ['Dry_Clover_g','Dry_Dead_g','Dry_Green_g','Dry_Total_g','GDM_g']\n\n# Make sure Sampling_Date is datetime\nif not np.issubdtype(train['Sampling_Date'].dtype, np.datetime64):\n    train['Sampling_Date'] = pd.to_datetime(train['Sampling_Date'])\n\n# Image resolver (handles common extensions) \ndef find_image_path(image_id, train_dir=TRAIN_DIR):\n    for ext in (\".jpg\",\".jpeg\",\".png\",\".bmp\",\".JPG\",\".JPEG\",\".PNG\",\".BMP\"):\n        p = train_dir / f\"{image_id}{ext}\"\n        if p.exists(): return p\n    return None\n\n# Thumbnail grid \ndef show_grid(df, title, rows=2, cols=4, seed=42, label_fn=None):\n    subset = df.sample(min(rows*cols, len(df)), random_state=seed)\n    plt.figure(figsize=(cols*3, rows*3))\n    k = 0\n    for _, r in subset.iterrows():\n        p = find_image_path(r[IDCOL])\n        if p is None: continue\n        img = cv2.cvtColor(cv2.imread(str(p), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)\n        img = cv2.resize(img, (256,256))\n        k += 1\n        plt.subplot(rows, cols, k)\n        plt.imshow(img); plt.axis(\"off\")\n        if label_fn:\n            plt.title(label_fn(r), fontsize=9)\n    plt.suptitle(title, y=1.02)\n    plt.tight_layout(); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:38.025717Z","iopub.execute_input":"2025-11-01T22:42:38.026018Z","iopub.status.idle":"2025-11-01T22:42:38.040675Z","shell.execute_reply.started":"2025-11-01T22:42:38.025997Z","shell.execute_reply":"2025-11-01T22:42:38.039774Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### “By State → which Species?” (counts + example images)","metadata":{}},{"cell_type":"code","source":"# Counts table\ncnt = (train.groupby([\"State\",\"Species\"])\n             .size().reset_index(name=\"n\")\n             .sort_values([\"State\",\"n\"], ascending=[True, False]))\ndisplay(cnt.head(20))\n\n# Montage per State with the most common Species labels\nfor state, g in cnt.groupby(\"State\"):\n    species_order = g.sort_values(\"n\", ascending=False)[\"Species\"].tolist()\n    df = train[train[\"State\"].eq(state) & train[\"Species\"].isin(species_order[:4])]\n    show_grid(df, f\"State={state} — sample Species\", rows=2, cols=4,\n              label_fn=lambda r: f\"{r['Species']}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:38.041582Z","iopub.execute_input":"2025-11-01T22:42:38.042058Z","iopub.status.idle":"2025-11-01T22:42:43.333275Z","shell.execute_reply.started":"2025-11-01T22:42:38.042028Z","shell.execute_reply":"2025-11-01T22:42:43.332055Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### “By Sampling Date” (timeline sampling)","metadata":{}},{"cell_type":"code","source":"# Month buckets to reduce sparsity\ntrain[\"_month\"] = train[\"Sampling_Date\"].dt.to_period(\"M\").astype(str)\n\n# counts per month\ndisplay(train[\"_month\"].value_counts().sort_index())\n\n# montage: sample per month (latest 6 months present)\nfor m in sorted(train[\"_month\"].unique())[-6:]:\n    df = train[train[\"_month\"].eq(m)]\n    if len(df) == 0: continue\n    show_grid(df, f\"Sampling month {m}\", rows=2, cols=4,\n              label_fn=lambda r: f\"{r['Species']} • {r['State']}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T22:42:43.334474Z","iopub.execute_input":"2025-11-01T22:42:43.334765Z","iopub.status.idle":"2025-11-01T22:42:50.628391Z","shell.execute_reply.started":"2025-11-01T22:42:43.334744Z","shell.execute_reply":"2025-11-01T22:42:50.627103Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### NDVI & Height: bins → thumbnails","metadata":{}},{"cell_type":"code","source":"# NDVI bins\nif \"Pre_GSHH_NDVI\" in train.columns:\n    train[\"_ndvi_bin\"] = pd.qcut(train[\"Pre_GSHH_NDVI\"], q=4, labels=False, duplicates=\"drop\")\n    for b in sorted(train[\"_ndvi_bin\"].dropna().unique()):\n        df = train[train[\"_ndvi_bin\"]==b]\n        show_grid(df, f\"NDVI quartile {int(b)} • NDVI~[{df['Pre_GSHH_NDVI'].min():.2f},{df['Pre_GSHH_NDVI'].max():.2f}]\",\n                  rows=2, cols=4,\n                  label_fn=lambda r: f\"NDVI {r['Pre_GSHH_NDVI']:.2f} • Total {r['Dry_Total_g']:.0f}g\")\n\n# Height bins\nif \"Height_Ave_cm\" in train.columns:\n    train[\"_h_bin\"] = pd.qcut(train[\"Height_Ave_cm\"], q=4, labels=False, duplicates=\"drop\")\n    for b in sorted(train[\"_h_bin\"].dropna().unique()):\n        df = train[train[\"_h_bin\"]==b]\n        show_grid(df, f\"Height quartile {int(b)} • H~[{df['Height_Ave_cm'].min():.1f},{df['Height_Ave_cm'].max():.1f}]\",\n                  rows=2, cols=4,\n                  label_fn=lambda r: f\"H {r['Height_Ave_cm']:.1f}cm • Total {r['Dry_Total_g']:.0f}g\")\n\n# cleanup  cols\ntrain.drop(columns=[\"_month\",\"_ndvi_bin\",\"_h_bin\"], errors=\"ignore\", inplace=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:28.377535Z","iopub.execute_input":"2025-11-01T23:03:28.378237Z","iopub.status.idle":"2025-11-01T23:03:38.500188Z","shell.execute_reply.started":"2025-11-01T23:03:28.378210Z","shell.execute_reply":"2025-11-01T23:03:38.498803Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Quick numeric EDA alongside thumbnails","metadata":{}},{"cell_type":"code","source":"# Distributions\nax = train[TARGETS].plot(kind=\"kde\", figsize=(8,4), title=\"Target distributions (raw)\")\nax.set_xlim(left=0); plt.show()\n\n# Correlations among targets and with NDVI/Height (if present)\nnum_cols = TARGETS + [c for c in [\"Pre_GSHH_NDVI\",\"Height_Ave_cm\"] if c in train.columns]\ncorr = train[num_cols].corr()\ndisplay(corr.style.background_gradient(cmap=\"RdBu_r\", vmin=-1, vmax=1).format(\"{:.2f}\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.501420Z","iopub.execute_input":"2025-11-01T23:03:38.501669Z","iopub.status.idle":"2025-11-01T23:03:38.884353Z","shell.execute_reply.started":"2025-11-01T23:03:38.501628Z","shell.execute_reply":"2025-11-01T23:03:38.883593Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 3. Augmentations\n\nWe use a light but effective set suitable for pasture canopies:\n- RandomResizedCrop / Resize\n- Horizontal/vertical flip (pasture has no canonical orientation)\n- Color jitter & CLAHE (handle lighting/contrast variability)\n- Small rotation / shift / scale\n- Cutout (optional)\n\n","metadata":{}},{"cell_type":"code","source":"import albumentations as A\nfrom albumentations.pytorch import ToTensorV2\n\ndef _RRC(sz, **kwargs):\n    \"\"\"RandomResizedCrop that works with Albumentations v1 or v2.\"\"\"\n    try:\n        # v2: expects size=(H, W)\n        return A.RandomResizedCrop(size=(sz, sz), **kwargs)\n    except Exception:\n        # v1: expects height, width\n        return A.RandomResizedCrop(height=sz, width=sz, **kwargs)\n\ndef _Resize(sz):\n    \"\"\"Resize that works with Albumentations v1 or v2.\"\"\"\n    try:\n        return A.Resize(size=(sz, sz))\n    except Exception:\n        return A.Resize(height=sz, width=sz)\n\ndef get_transforms(img_size, is_train=True):\n    if is_train:\n        return A.Compose([\n            _RRC(img_size, scale=(0.8, 1.0)),\n            A.HorizontalFlip(p=0.5),\n            A.VerticalFlip(p=0.3),\n            A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.1, rotate_limit=10, p=0.5),\n            A.RandomBrightnessContrast(p=0.4),\n            A.CLAHE(p=0.2),\n            A.CoarseDropout(max_holes=4,\n                            max_height=int(img_size*0.08),\n                            max_width=int(img_size*0.08),\n                            p=0.3),\n            A.Normalize(),\n            ToTensorV2(),\n        ])\n    else:\n        return A.Compose([\n            _Resize(img_size),\n            A.Normalize(),\n            ToTensorV2(),\n        ])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.885135Z","iopub.execute_input":"2025-11-01T23:03:38.885331Z","iopub.status.idle":"2025-11-01T23:03:38.892700Z","shell.execute_reply.started":"2025-11-01T23:03:38.885316Z","shell.execute_reply":"2025-11-01T23:03:38.892021Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 4. Dataset / DataLoader\n","metadata":{}},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    def __init__(self, df, img_root=None, transform=None, targets=TARGETS, use_abs=False):\n        self.df = df.reset_index(drop=True)\n        self.img_root = Path(img_root) if img_root else None\n        self.transform = transform\n        self.targets = targets\n        self.idcol = IDCOL\n        self.use_abs = use_abs\n\n    def _find_path(self, row):\n        if self.use_abs and 'image_path' in row:\n            p = Path(row['image_path'])\n            if p.exists(): return p\n        img_id = row[self.idcol]\n        if self.img_root is None: return None\n        for ext in ('.jpg','.jpeg','.png','.bmp'):\n            p = self.img_root/f\"{img_id}{ext}\"\n            if p.exists(): return p\n        return None\n\n    def __len__(self): return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        p = self._find_path(row)\n        if p is None:\n            raise FileNotFoundError(row.get(self.idcol, row.get('image_path', 'UNKNOWN_ID')))\n        img = cv2.cvtColor(cv2.imread(str(p), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)\n        if self.transform: img = self.transform(image=img)['image']\n        y = torch.tensor(row[self.targets].values.astype('float32'))\n        return img, y, row[self.idcol]\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.894380Z","iopub.execute_input":"2025-11-01T23:03:38.894613Z","iopub.status.idle":"2025-11-01T23:03:38.910582Z","shell.execute_reply.started":"2025-11-01T23:03:38.894596Z","shell.execute_reply":"2025-11-01T23:03:38.909742Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 5. Train / Val split\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\n# Stratify by Dry_Total_g quantiles to balance difficulty\nbins = pd.qcut(train['Dry_Total_g'], q=min(10, train['Dry_Total_g'].nunique()), labels=False, duplicates='drop')\ntrain['strat'] = bins\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=CFG['seed'])\ntrain['fold'] = -1\nfor i,(tr,va) in enumerate(skf.split(train, train['strat'])):\n    train.loc[va,'fold'] = i\ndisplay(train['fold'].value_counts())\n\nFOLD = 0 \ntrn_df = train[train['fold']!=FOLD].copy()\nval_df = train[train['fold']==FOLD].copy()\n\ntrn_ds = BiomassDataset(trn_df, TRAIN_IMG_DIR, get_transforms(CFG['img_size'], True), use_abs=USE_ABS)\nval_ds = BiomassDataset(val_df, TRAIN_IMG_DIR, get_transforms(CFG['img_size'], False), use_abs=USE_ABS)\n\ntrn_loader = DataLoader(trn_ds, batch_size=CFG['batch_size'], shuffle=True,  num_workers=CFG['num_workers'], pin_memory=True)\nval_loader = DataLoader(val_ds, batch_size=CFG['batch_size']*2, shuffle=False, num_workers=CFG['num_workers'], pin_memory=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.911373Z","iopub.execute_input":"2025-11-01T23:03:38.911911Z","iopub.status.idle":"2025-11-01T23:03:38.961604Z","shell.execute_reply.started":"2025-11-01T23:03:38.911874Z","shell.execute_reply":"2025-11-01T23:03:38.960905Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 6. Model (timm EfficientNet, multi-head regression)\n\n- Backbone: `tf_efficientnetv2_s_in21k`\n- Head: Linear to 5 outputs\n- Loss: `SmoothL1Loss` on log1p targets \n- Metric: Weighted $R^2$ \n","metadata":{}},{"cell_type":"code","source":"\ndef r2_score_torch(y_true, y_pred, eps=1e-9):\n    y_true_mean = torch.mean(y_true, dim=0)\n    ss_tot = torch.sum((y_true - y_true_mean)**2, dim=0)\n    ss_res = torch.sum((y_true - y_pred)**2, dim=0)\n    r2 = 1 - ss_res / (ss_tot + eps)\n    return r2\n\nWEIGHTS = torch.tensor([0.1,0.1,0.1,0.2,0.5], dtype=torch.float32)\n\nclass BiomassModel(pl.LightningModule):\n    def __init__(self, backbone=CFG['backbone'], lr=CFG['lr'], wd=CFG['weight_decay']):\n        super().__init__()\n        self.save_hyperparameters()\n        self.backbone = timm.create_model(backbone, pretrained=False, num_classes=0, global_pool=\"avg\")\n        in_features = self.backbone.num_features\n        self.head = nn.Linear(in_features, len(TARGETS))\n        self.loss_fn = nn.SmoothL1Loss()\n        self.lr = lr; self.wd = wd\n\n    def forward(self, x):\n        f = self.backbone(x)\n        out = self.head(f)\n        return out\n\n    def common_step(self, batch, stage='train'):\n        x, y, _ = batch\n        y_t = torch.log1p(y)\n        y_hat = self(x)\n        loss = self.loss_fn(y_hat, y_t)\n\n        with torch.no_grad():\n            r2s = r2_score_torch(y_t, y_hat)\n            weighted = (r2s * WEIGHTS.to(self.device)).sum().item()\n        self.log(f\"{stage}_loss\", loss, prog_bar=True)\n        self.log(f\"{stage}_r2w\", weighted, prog_bar=True)\n        return loss\n\n    def training_step(self, batch, batch_idx):\n        return self.common_step(batch, 'train')\n\n    def validation_step(self, batch, batch_idx):\n        self.common_step(batch, 'val')\n\n    def configure_optimizers(self):\n        opt = torch.optim.AdamW(self.parameters(), lr=self.lr, weight_decay=self.wd)\n        # cosine with warmup\n        warmup = 1  # epochs\n        total = CFG['epochs']\n        def lr_lambda(epoch):\n            if epoch < warmup:  # linear warmup\n                return float(epoch + 1) / float(max(1, warmup))\n            # cosine decay  [warmup..total)\n            progress = (epoch - warmup) / float(max(1, total - warmup))\n            return 0.5 * (1.0 + math.cos(math.pi * progress))\n        sch = torch.optim.lr_scheduler.LambdaLR(opt, lr_lambda=lr_lambda)\n        return {'optimizer': opt, 'lr_scheduler': sch}\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.962411Z","iopub.execute_input":"2025-11-01T23:03:38.962725Z","iopub.status.idle":"2025-11-01T23:03:38.976301Z","shell.execute_reply.started":"2025-11-01T23:03:38.962704Z","shell.execute_reply":"2025-11-01T23:03:38.975500Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 7. Train\n","metadata":{}},{"cell_type":"code","source":"\nmodel = BiomassModel()\n\nckpt_cb = ModelCheckpoint(\n    dirpath=str(OUTPUT_DIR),\n    filename=\"model-{epoch:02d}-{val_r2w:.4f}\",\n    monitor=\"val_r2w\",\n    mode=\"max\",\n    save_top_k=1\n)\nlr_cb = LearningRateMonitor(logging_interval='epoch')\nes_cb = EarlyStopping(monitor=\"val_r2w\", mode=\"max\", patience=3)\n\nlogger = CSVLogger(save_dir=str(OUTPUT_DIR), name=\"logs\")\n\ntrainer = pl.Trainer(\n    max_epochs=CFG['epochs'],\n    precision=CFG['precision'],\n    callbacks=[ckpt_cb, lr_cb, es_cb],\n    logger=logger,\n    default_root_dir=str(OUTPUT_DIR),\n    gradient_clip_val=1.0,\n    deterministic=False,\n    accumulate_grad_batches=1,\n)\n\ntrainer.fit(model, trn_loader, val_loader)\n\nbest_ckpt = ckpt_cb.best_model_path\nprint(\"Best ckpt:\", best_ckpt)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:03:38.977184Z","iopub.execute_input":"2025-11-01T23:03:38.977484Z","iopub.status.idle":"2025-11-01T23:05:45.182826Z","shell.execute_reply.started":"2025-11-01T23:03:38.977456Z","shell.execute_reply":"2025-11-01T23:05:45.181873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Plot train vs val metric over epochs + their averages\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom pathlib import Path\n\n#Locate the metrics.csv that CSVLogger just wrote\ntry:\n    log_dir = Path(logger.log_dir)  # preferred (available after trainer.fit)\nexcept Exception:\n    # fallback: find the newest logs/version_*\n    versions = sorted((Path(OUTPUT_DIR) / \"logs\").glob(\"version_*\"))\n    assert versions, \"No CSVLogger versions found under OUTPUT_DIR/logs\"\n    log_dir = versions[-1]\nmetrics_csv = log_dir / \"metrics.csv\"\nassert metrics_csv.exists(), f\"metrics.csv not found at {metrics_csv}\"\n\nmetrics = pd.read_csv(metrics_csv)\n\n# Pick the metric columns (robust to name variants)\n# Change METRIC_KEY if your model logs a different name.\nMETRIC_KEY = \"r2w\"  # e.g., 'r2w' or 'loss' or 'rmse'\ntrain_col_candidates = [f\"train_{METRIC_KEY}\", \"train_r2w\", \"train_r2\", \"train_score\", \"train_loss\"]\nval_col_candidates   = [f\"val_{METRIC_KEY}\",   \"val_r2w\",   \"val_r2\",   \"val_score\",   \"val_loss\"]\n\ndef first_present(cols):\n    for c in cols:\n        if c in metrics.columns:\n            return c\n    return None\n\nTRAIN_COL = first_present(train_col_candidates)\nVAL_COL   = first_present(val_col_candidates)\nassert TRAIN_COL is not None and VAL_COL is not None, f\"No train/val metric columns found in {metrics.columns.tolist()}\"\n\n# Reduce to one value per epoch (last logged row for that epoch)\ndef per_epoch_last(df, col):\n    d = df[[\"epoch\", col]].dropna()\n    # keep the last record per epoch\n    d = d.groupby(\"epoch\", as_index=False).last()\n    return d.rename(columns={col: \"value\"})\n\ntrain_e = per_epoch_last(metrics, TRAIN_COL)\nval_e   = per_epoch_last(metrics, VAL_COL)\n\n# Align epochs (outer-join, in case early epochs log train but not val or vice versa)\nplot_df = pd.merge(train_e.rename(columns={\"value\": \"train\"}),\n                   val_e.rename(columns={\"value\": \"val\"}),\n                   on=\"epoch\", how=\"outer\").sort_values(\"epoch\")\nplot_df.reset_index(drop=True, inplace=True)\n\n# Compute averages across all available epochs\ntrain_avg = np.nanmean(plot_df[\"train\"].values)\nval_avg   = np.nanmean(plot_df[\"val\"].values)\n\n# Plot\nplt.figure(figsize=(8.5, 5.0))\nplt.plot(plot_df[\"epoch\"], plot_df[\"train\"], marker=\"o\", label=f\"Train ({TRAIN_COL})\")\nplt.plot(plot_df[\"epoch\"], plot_df[\"val\"],   marker=\"s\", label=f\"Val ({VAL_COL})\")\nplt.axhline(train_avg, linestyle=\"--\", linewidth=1.2, label=f\"Train avg = {train_avg:.4f}\")\nplt.axhline(val_avg,   linestyle=\"--\", linewidth=1.2, label=f\"Val avg = {val_avg:.4f}\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(METRIC_KEY.upper())\nplt.title(\"Training vs Validation metric per epoch\")\nplt.grid(True, alpha=0.25)\nplt.legend()\nplt.tight_layout()\nplt.show()\n\n# 6) (Optional) print a small table\ndisplay(plot_df.assign(train_avg=train_avg, val_avg=val_avg))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:05:45.184073Z","iopub.execute_input":"2025-11-01T23:05:45.184398Z","iopub.status.idle":"2025-11-01T23:05:45.507339Z","shell.execute_reply.started":"2025-11-01T23:05:45.184364Z","shell.execute_reply":"2025-11-01T23:05:45.506292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n## 8. Inference (+ optional TTA) and Submission\n\n- Loads the best checkpoint\n- Predicts on **test.csv**\n- Applies `expm1` to invert the training transform\n- Writes `submission.csv` with 5 target columns\n","metadata":{}},{"cell_type":"code","source":"# ===========================================\n# FINAL SUBMISSION (uses test.csv for mapping,\n# sample_submission.csv for order)\n# ===========================================\nimport os, json, hashlib\nfrom pathlib import Path\nimport numpy as np, pandas as pd, torch, cv2\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm\n\nCOMP = os.environ.get(\"KAGGLE_COMPETITION\", \"csiro-biomass\")\nROOT = Path(f\"/kaggle/input/{COMP}\")\nassert ROOT.exists(), f\"Competition input not mounted: {ROOT}\"\n\n#Load runtime files\ntest_rt = pd.read_csv(ROOT / \"test.csv\")\nsub_tmpl = pd.read_csv(ROOT / \"sample_submission.csv\")\n\nassert {\"sample_id\",\"image_path\",\"target_name\"}.issubset(test_rt.columns), test_rt.columns.tolist()\nassert list(sub_tmpl.columns) == [\"sample_id\",\"target\"], sub_tmpl.columns.tolist()\n\n# Canonical targets for indexing model outputs\nTARGETS = ['Dry_Green_g','Dry_Dead_g','Dry_Clover_g','GDM_g','Dry_Total_g']\ntgt2idx = {t:i for i,t in enumerate(TARGETS)}\n\n# Resolve absolute paths for unique images\ndef resolve_path(image_path):\n    p = ROOT / image_path\n    if p.exists():\n        return p\n    stem = Path(image_path).stem\n    for ext in (\".jpg\",\".jpeg\",\".png\",\".bmp\",\".JPG\",\".JPEG\",\".PNG\",\".BMP\"):\n        q = ROOT / \"test\" / f\"{stem}{ext}\"\n        if q.exists():\n            return q\n    return p  # may not exist locally; will exist at scoring\n\nuniq = test_rt[[\"image_path\"]].drop_duplicates().copy()\nuniq[\"image_id\"] = uniq[\"image_path\"].apply(lambda s: Path(str(s)).stem)\nuniq[\"abs_path\"] = uniq[\"image_path\"].apply(resolve_path)\n\n# Minimal dataset for unique images\nclass _UDS(Dataset):\n    def __init__(self, df): self.df = df.reset_index(drop=True)\n    def __len__(self): return len(self.df)\n    def __getitem__(self, i):\n        row = self.df.iloc[i]\n        p = row[\"abs_path\"]\n        img = cv2.cvtColor(cv2.imread(str(p), cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)\n        return img, row[\"image_id\"]\n\nuds = _UDS(uniq)\nldr = DataLoader(\n    uds, batch_size=CFG['batch_size']*2, shuffle=False, num_workers=CFG['num_workers'],\n    collate_fn=lambda b: tuple(zip(*b))\n)\n\n# Model & transforms\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nmodel = BiomassModel.load_from_checkpoint(best_ckpt, backbone=CFG['backbone'], map_location='cpu').to(device).eval()\nbase_tf = get_transforms(CFG['img_size'], False)\ntrain_tf = get_transforms(CFG['img_size'], True)\n\ndef prep(imgs, tfm):\n    xs = [tfm(image=im)['image'] for im in imgs]\n    return torch.stack(xs).to(device, non_blocking=True)\n\n# Predict once per unique image (TTA optional)\nid2vec = {}\nwith torch.no_grad():\n    for imgs, ids_batch in tqdm(ldr, desc=\"Predict unique images\"):\n        xb = prep(list(imgs), base_tf)\n        ys = [model(xb)]\n        for _ in range(max(0, CFG.get('tta',1)-1)):\n            xa = prep(list(imgs), train_tf)\n            ys.append(model(xa))\n        y = torch.stack(ys).mean(0).cpu().numpy()  # [B,5] log-space\n        y = np.expm1(y)                             # back to grams\n        for k, iid in enumerate(ids_batch):\n            id2vec[iid] = y[k]\n\n# Build long predictions in test.csv order, then map to sample_id\nvals = []\nfor _, r in test_rt.iterrows():\n    iid = Path(str(r[\"image_path\"])).stem\n    tname = str(r[\"target_name\"]).strip()\n    vec = id2vec.get(iid)\n    j = tgt2idx.get(tname)\n    v = 0.0\n    if (vec is not None) and (j is not None):\n        v = float(vec[j])\n        if not np.isfinite(v) or v < 0: v = 0.0\n    vals.append(v)\n\npred_long = pd.DataFrame({\n    \"sample_id\": test_rt[\"sample_id\"],\n    \"target\": vals\n})\n\n# Left-join onto the runtime template to guarantee exact row count & order\nfinal_sub = sub_tmpl[[\"sample_id\"]].merge(pred_long, on=\"sample_id\", how=\"left\")\n# Fill any missing with 0.0 (shouldn't happen if everything resolved)\nfinal_sub[\"target\"] = final_sub[\"target\"].fillna(0.0).astype(\"float64\")\n\n# Strict validation & save\nassert list(final_sub.columns) == [\"sample_id\",\"target\"]\nassert len(final_sub) == len(sub_tmpl), f\"Row count mismatch {len(final_sub)} vs {len(sub_tmpl)}\"\nassert set(final_sub[\"sample_id\"]) == set(sub_tmpl[\"sample_id\"])\nassert not final_sub[\"sample_id\"].duplicated().any()\nassert np.isfinite(final_sub[\"target\"]).all()\n\nPath(\"./outputs\").mkdir(parents=True, exist_ok=True)\nfinal_sub.to_csv(\"./outputs/submission.csv\", index=False)\nfinal_sub.to_csv(\"submission.csv\", index=False)\n\ndef _md5(p):\n    h = hashlib.md5()\n    with open(p,\"rb\") as f:\n        for ch in iter(lambda: f.read(8192), b\"\"): h.update(ch)\n    return h.hexdigest()\n\nprint(json.dumps({\n    \"rows\": int(len(final_sub)),\n    \"md5\": _md5(\"submission.csv\"),\n    \"head\": final_sub.head(3).to_dict(orient=\"records\")\n}, indent=2))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-01T23:05:45.508383Z","iopub.execute_input":"2025-11-01T23:05:45.508737Z","iopub.status.idle":"2025-11-01T23:05:47.162031Z","shell.execute_reply.started":"2025-11-01T23:05:45.508714Z","shell.execute_reply":"2025-11-01T23:05:47.161030Z"}},"outputs":[],"execution_count":null}]}