{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":14296488,"sourceType":"datasetVersion","datasetId":9125936},{"sourceId":4533,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3325,"modelId":986},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3329,"modelId":986}],"dockerImageVersionId":31234,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# EDA\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom PIL import Image\nimport random\nimport plotly.express as px\nfrom plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nimport numpy as np\nimport os\n\n# DINO-V2\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data import Dataset, DataLoader\nimport torchvision.transforms as T\nfrom transformers import AutoModel, AutoImageProcessor\n\nfrom sklearn.model_selection import KFold","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.636872Z","iopub.execute_input":"2025-12-27T23:44:55.637697Z","iopub.status.idle":"2025-12-27T23:44:55.643415Z","shell.execute_reply.started":"2025-12-27T23:44:55.637656Z","shell.execute_reply":"2025-12-27T23:44:55.642686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/csiro-biomass\"\nMODEL_PATH = \"/kaggle/input/dinov2/pytorch/small/1\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.644804Z","iopub.execute_input":"2025-12-27T23:44:55.645137Z","iopub.status.idle":"2025-12-27T23:44:55.663895Z","shell.execute_reply.started":"2025-12-27T23:44:55.645106Z","shell.execute_reply":"2025-12-27T23:44:55.663269Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(f\"{DATA_PATH}/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.664711Z","iopub.execute_input":"2025-12-27T23:44:55.665079Z","iopub.status.idle":"2025-12-27T23:44:55.688017Z","shell.execute_reply.started":"2025-12-27T23:44:55.665044Z","shell.execute_reply":"2025-12-27T23:44:55.687330Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"print(f\"Total number of samples: {len(train_df) / 5}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.689390Z","iopub.execute_input":"2025-12-27T23:44:55.689651Z","iopub.status.idle":"2025-12-27T23:44:55.701294Z","shell.execute_reply.started":"2025-12-27T23:44:55.689622Z","shell.execute_reply":"2025-12-27T23:44:55.700674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.720599Z","iopub.execute_input":"2025-12-27T23:44:55.720817Z","iopub.status.idle":"2025-12-27T23:44:55.732125Z","shell.execute_reply.started":"2025-12-27T23:44:55.720795Z","shell.execute_reply":"2025-12-27T23:44:55.731328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"Number of species: {len(train_df[\"Species\"].unique())}\")\nprint(f\"Number of states: {len(train_df[\"State\"].unique())}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.733802Z","iopub.execute_input":"2025-12-27T23:44:55.734149Z","iopub.status.idle":"2025-12-27T23:44:55.748410Z","shell.execute_reply.started":"2025-12-27T23:44:55.734121Z","shell.execute_reply":"2025-12-27T23:44:55.747628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"Sampling_Date\"] = pd.to_datetime(train_df[\"Sampling_Date\"])\n\ntrain_df[\"Sampling_Year\"] = train_df[\"Sampling_Date\"].dt.year\ntrain_df[\"Sampling_Mth\"] = train_df[\"Sampling_Date\"].dt.month\ntrain_df[\"Sampling_Day\"] = train_df[\"Sampling_Date\"].dt.day\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.749320Z","iopub.execute_input":"2025-12-27T23:44:55.749623Z","iopub.status.idle":"2025-12-27T23:44:55.770775Z","shell.execute_reply.started":"2025-12-27T23:44:55.749588Z","shell.execute_reply":"2025-12-27T23:44:55.770076Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Let's visualize some samples from the training set.","metadata":{}},{"cell_type":"markdown","source":"## Visualize Samples","metadata":{}},{"cell_type":"code","source":"# --- config ---\nN_SAMPLES = 9\nTARGET_ORDER = [\n    \"Dry_Green_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Clover_g\",\n    \"GDM_g\",\n    \"Dry_Total_g\",\n]\n\n# --- sample unique images ---\nimage_ids = train_df[\"image_path\"].unique()\nsampled_images = random.sample(list(image_ids), N_SAMPLES)\n\n# --- plot ---\nfig, axes = plt.subplots(3, 3, figsize=(20, 8))\naxes = axes.flatten()\n\nfor ax, img_path in zip(axes, sampled_images):\n    rows = train_df[train_df[\"image_path\"] == img_path]\n\n    # load image\n    img = Image.open(f\"{DATA_PATH}/{img_path}\").convert(\"RGB\")\n    ax.imshow(img)\n    ax.axis(\"off\")\n\n    # collect targets\n    targets = (\n        rows.set_index(\"target_name\")[\"target\"]\n        .reindex(TARGET_ORDER)\n        .round(2)\n    )\n\n    title = \"\\n\".join([f\"{k}: {v}\" for k, v in targets.items()])\n    ax.set_title(title, fontsize=9)\n\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:55.771865Z","iopub.execute_input":"2025-12-27T23:44:55.772212Z","iopub.status.idle":"2025-12-27T23:44:57.705282Z","shell.execute_reply.started":"2025-12-27T23:44:55.772174Z","shell.execute_reply":"2025-12-27T23:44:57.704360Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# columns that are constant per image\nMETA_COLS = [\n    \"image_path\",\n    \"Sampling_Mth\",\n    \"State\",\n    \"Species\",\n    \"Pre_GSHH_NDVI\",\n    \"Height_Ave_cm\"\n]\n\nmeta = train_df[META_COLS].drop_duplicates()\n\ntargets = train_df.pivot(\n    index=\"image_path\",\n    columns=\"target_name\",\n    values=\"target\"\n)\n\ndf_wide = meta.merge(\n    targets,\n    left_on=\"image_path\",\n    right_index=True\n)\n\ndf_wide.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:57.708053Z","iopub.execute_input":"2025-12-27T23:44:57.708405Z","iopub.status.idle":"2025-12-27T23:44:57.737006Z","shell.execute_reply.started":"2025-12-27T23:44:57.708346Z","shell.execute_reply":"2025-12-27T23:44:57.736258Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"It seems like GDM_g = Dry_Green_g + Dry_Clover_g and \nDry_Total_g = GDM_g + Dry_Dead_g","metadata":{}},{"cell_type":"markdown","source":"## Marginal Distribution of Targets","metadata":{}},{"cell_type":"code","source":"fig = make_subplots(\n    rows=1, cols=5,\n    subplot_titles=(\"Clover\", \"Dead\", \"Green\", \"Total\", \"GDM\"),\n    shared_yaxes=True\n)\n\nfig.add_trace(\n    go.Histogram(x=df_wide[\"Dry_Clover_g\"], nbinsx=50),\n    row=1, col=1\n)\n\nfig.add_trace(\n    go.Histogram(x=df_wide[\"Dry_Dead_g\"], nbinsx=50),\n    row=1, col=2\n)\n\nfig.add_trace(\n    go.Histogram(x=df_wide[\"Dry_Green_g\"], nbinsx=50),\n    row=1, col=3\n)\n\nfig.add_trace(\n    go.Histogram(x=df_wide[\"Dry_Total_g\"], nbinsx=50),\n    row=1, col=4\n)\n\nfig.add_trace(\n    go.Histogram(x=df_wide[\"GDM_g\"], nbinsx=50),\n    row=1, col=5\n)\n\nfig.update_layout(\n    title_text=\"Target Distributions\",\n    showlegend=False,\n    height=400,\n    width=1200,\n    yaxis_title = \"Frequency\",\n    xaxis_title = \"Grams (g)\"\n)\n\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:57.738016Z","iopub.execute_input":"2025-12-27T23:44:57.738352Z","iopub.status.idle":"2025-12-27T23:44:57.794469Z","shell.execute_reply.started":"2025-12-27T23:44:57.738316Z","shell.execute_reply":"2025-12-27T23:44:57.793796Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Distribution of Species/State","metadata":{}},{"cell_type":"code","source":"species_bar = px.bar(df_wide, y = \"Species\", color = \"State\")\nspecies_bar.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:57.795485Z","iopub.execute_input":"2025-12-27T23:44:57.795725Z","iopub.status.idle":"2025-12-27T23:44:57.851499Z","shell.execute_reply.started":"2025-12-27T23:44:57.795703Z","shell.execute_reply":"2025-12-27T23:44:57.850830Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Except for Clover, Ryegrass, and Ryegrass_Clover, species tend to appear in only one state.","metadata":{}},{"cell_type":"markdown","source":"##  Correlation Matrix","metadata":{}},{"cell_type":"code","source":"CATEGORICAL_COLS = [\"State\", \"Species\", \"Sampling_Mth\"]\nNUMERIC_COLS = [\"Pre_GSHH_NDVI\", \"Height_Ave_cm\"]\nTARGET_COLS = [\n    \"Dry_Clover_g\",\n    \"Dry_Dead_g\",\n    \"Dry_Green_g\",\n    \"Dry_Total_g\",\n    \"GDM_g\",\n]\n\n# one-hot encode categoricals\ndf_full = pd.get_dummies(\n    df_wide[\n        CATEGORICAL_COLS + NUMERIC_COLS + TARGET_COLS\n    ],\n    columns=CATEGORICAL_COLS,\n    drop_first=False\n)\n\n# reorder columns for readability\nordered_cols = (\n    NUMERIC_COLS +\n    sorted([c for c in df_full.columns if c.startswith(\"Sampling_Mth_\")]) +\n    sorted([c for c in df_full.columns if c.startswith(\"State_\")]) +\n    sorted([c for c in df_full.columns if c.startswith(\"Species_\")]) +\n    TARGET_COLS\n)\n\ndf_full = df_full[ordered_cols]\n\n# compute correlation matrix\ncorr = df_full.corr()\n\nmask = np.triu(np.ones_like(corr, dtype=bool))\n\nplt.figure(figsize=(16, 14))\nsns.heatmap(\n    corr,\n    mask=mask,\n    cmap=\"coolwarm\",\n    center=0,\n    square=True,\n    cbar_kws={\"shrink\": 0.7}\n)\nplt.title(\"Full Feature Correlation Matrix (One-Hot Encoded)\")\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:57.852406Z","iopub.execute_input":"2025-12-27T23:44:57.852619Z","iopub.status.idle":"2025-12-27T23:44:58.551626Z","shell.execute_reply.started":"2025-12-27T23:44:57.852597Z","shell.execute_reply":"2025-12-27T23:44:58.550771Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Tabular Feature Predictions (RF for each target separately)","metadata":{}},{"cell_type":"code","source":"df_wide.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:58.552654Z","iopub.execute_input":"2025-12-27T23:44:58.553014Z","iopub.status.idle":"2025-12-27T23:44:58.569950Z","shell.execute_reply.started":"2025-12-27T23:44:58.552978Z","shell.execute_reply":"2025-12-27T23:44:58.568958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:58.571191Z","iopub.execute_input":"2025-12-27T23:44:58.571686Z","iopub.status.idle":"2025-12-27T23:44:58.586195Z","shell.execute_reply.started":"2025-12-27T23:44:58.571638Z","shell.execute_reply":"2025-12-27T23:44:58.585423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df_wide[[\"image_path\"] + CATEGORICAL_COLS + NUMERIC_COLS]\ny = df_wide[TARGET_COLS]\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state = 67)\n\nX_train_ids = X_train[\"image_path\"].str.split(\"/\").str[1].str.split(\".\").str[0]\nX_val_ids = X_val[\"image_path\"].str.split(\"/\").str[1].str.split(\".\").str[0]\n\nX_train.drop(columns=[\"image_path\"])\nX_val.drop(columns=[\"image_path\"])\n\ny_train_clover = y_train[[\"Dry_Clover_g\"]]\ny_val_clover  = y_val[[\"Dry_Clover_g\"]]\n\ny_train_dead   = y_train[[\"Dry_Dead_g\"]]\ny_val_dead    = y_val[[\"Dry_Dead_g\"]]\n\ny_train_green  = y_train[[\"Dry_Green_g\"]]\ny_val_green   = y_val[[\"Dry_Green_g\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:58.587286Z","iopub.execute_input":"2025-12-27T23:44:58.587633Z","iopub.status.idle":"2025-12-27T23:44:58.613405Z","shell.execute_reply.started":"2025-12-27T23:44:58.587596Z","shell.execute_reply":"2025-12-27T23:44:58.612624Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocess = ColumnTransformer(\n    transformers=[\n        (\"cat\", OneHotEncoder(handle_unknown=\"ignore\"), CATEGORICAL_COLS),\n        (\"num\", \"passthrough\", NUMERIC_COLS),\n    ]\n)\n\nX_train_processed = preprocess.fit_transform(X_train)\nX_val_processed = preprocess.transform(X_val)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:58.614367Z","iopub.execute_input":"2025-12-27T23:44:58.614669Z","iopub.status.idle":"2025-12-27T23:44:58.639343Z","shell.execute_reply.started":"2025-12-27T23:44:58.614638Z","shell.execute_reply":"2025-12-27T23:44:58.638509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"rf_clover = RandomForestRegressor()\nrf_dead = RandomForestRegressor()\nrf_green = RandomForestRegressor()\n\nrf_clover.fit(X_train_processed, y_train_clover)\nrf_dead.fit(X_train_processed, y_train_dead)\nrf_green.fit(X_train_processed, y_train_green)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:58.640393Z","iopub.execute_input":"2025-12-27T23:44:58.640652Z","iopub.status.idle":"2025-12-27T23:44:59.415960Z","shell.execute_reply.started":"2025-12-27T23:44:58.640618Z","shell.execute_reply":"2025-12-27T23:44:59.415107Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_wide_predictions(clover_preds, dead_preds, green_preds, ids):\n    GDM_preds = green_preds + clover_preds\n    total_preds = green_preds + clover_preds + dead_preds\n\n    pred_df = pd.DataFrame({\n        \"sample_id\": ids, \n        \"Dry_Green_g\": green_preds,\n        \"Dry_Dead_g\": dead_preds,\n        \"Dry_Clover_g\": clover_preds,\n        \"GDM_g\": GDM_preds,\n        \"Dry_Total_g\": total_preds,\n    })\n\n    return pred_df\n\ndef wide_to_long_preds(pred_df):\n    submission_df = (\n        pred_df\n        .rename(columns={\"sample_id\": \"image_id\"})\n        .melt(id_vars=\"image_id\", var_name=\"target_name\", value_name=\"target\")\n        .assign(sample_id=lambda d: d.image_id + \"__\" + d.target_name)\n        [[\"sample_id\", \"target\"]]\n    )\n\n    return submission_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.418571Z","iopub.execute_input":"2025-12-27T23:44:59.418824Z","iopub.status.idle":"2025-12-27T23:44:59.424389Z","shell.execute_reply.started":"2025-12-27T23:44:59.418799Z","shell.execute_reply":"2025-12-27T23:44:59.423732Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_preds = wide_to_long_preds(\n                create_wide_predictions(rf_clover.predict(X_val_processed), \n                                        rf_dead.predict(X_val_processed), \n                                        rf_green.predict(X_val_processed), \n                                        ids = X_val_ids))\n\ny_val[\"sample_id\"] = X_val_ids\n\nval_actual = wide_to_long_preds(y_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.425161Z","iopub.execute_input":"2025-12-27T23:44:59.425501Z","iopub.status.idle":"2025-12-27T23:44:59.479768Z","shell.execute_reply.started":"2025-12-27T23:44:59.425461Z","shell.execute_reply":"2025-12-27T23:44:59.479023Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def eval_R2(actual, predicted):\n    merged = actual.merge(predicted, on = \"sample_id\", suffixes = ('_actual', '_pred'))\n\n    weight_map = {\"Dry_Green_g\" : 0.1,\n                    \"Dry_Dead_g\": 0.1,\n                    \"Dry_Clover_g\": 0.1,\n                    \"GDM_g\": 0.2,\n                    \"Dry_Total_g\": 0.5\n                }\n    \n    merged[\"target_name\"] = merged[\"sample_id\"].str.split(\"__\").str[1]\n\n    merged[\"weight\"] = merged[\"target_name\"].map(weight_map)\n\n    w = merged[\"weight\"]\n    y_hat = merged[\"target_pred\"]\n    y = merged[\"target_actual\"]\n\n    y_bar = merged[\"target_actual\"].mean()\n\n    ss_res = np.sum(w * (y_hat - y) ** 2)\n    ss_tot = np.sum(w * (y_bar - y) ** 2)\n\n    r2 = 1 - ss_res / ss_tot\n\n    return r2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.480738Z","iopub.execute_input":"2025-12-27T23:44:59.481076Z","iopub.status.idle":"2025-12-27T23:44:59.486898Z","shell.execute_reply.started":"2025-12-27T23:44:59.481042Z","shell.execute_reply":"2025-12-27T23:44:59.486037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eval_R2(val_actual, val_preds)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.488050Z","iopub.execute_input":"2025-12-27T23:44:59.488475Z","iopub.status.idle":"2025-12-27T23:44:59.510655Z","shell.execute_reply.started":"2025-12-27T23:44:59.488438Z","shell.execute_reply":"2025-12-27T23:44:59.509889Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"So the tabular data available in the training data is not available at test-time. We will try using a model to extract these tabular features from the test data.","metadata":{}},{"cell_type":"markdown","source":"# DINO-V2","metadata":{}},{"cell_type":"markdown","source":"So what is DINO-V2? It is essentially an encoder that maps inputs to vector embeddings. It learns how to best represent an image or text in an embedding space. For DINO-V2, an output consists of a global CLS embedding defined for the entire image as well as a set of patch embeddings. A patch is just some p x p subimage. It is useful because we can learn characteristics about the biomass images from our training set using the tabular features available. To do this, we define an MLP that maps the CLS embeddings to the targets we want to predict. ","metadata":{}},{"cell_type":"code","source":"device = torch.device('cuda')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.526448Z","iopub.execute_input":"2025-12-27T23:44:59.526735Z","iopub.status.idle":"2025-12-27T23:44:59.540314Z","shell.execute_reply.started":"2025-12-27T23:44:59.526702Z","shell.execute_reply":"2025-12-27T23:44:59.539587Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_wide.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.541352Z","iopub.execute_input":"2025-12-27T23:44:59.541743Z","iopub.status.idle":"2025-12-27T23:44:59.564827Z","shell.execute_reply.started":"2025-12-27T23:44:59.541703Z","shell.execute_reply":"2025-12-27T23:44:59.564042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassDataset(Dataset):\n    # use df_wide\n    def __init__(self, df, transform):\n        self.df = df.reset_index(drop = True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        img_path = row[\"image_path\"]\n\n        img = Image.open(f\"{DATA_PATH}/{img_path}\").convert(\"RGB\")\n\n        img = self.transform(img)\n\n        # ---- biomass targets (5) ----\n        biomass = torch.tensor(\n            row[TARGET_COLS].to_numpy(dtype=float),\n            dtype=torch.float32\n        )\n\n        # ---- auxiliary targets ----\n        ndvi = torch.tensor(row[\"Pre_GSHH_NDVI\"], dtype = torch.float32)\n        height = torch.tensor(row[\"Height_Ave_cm\"], dtype = torch.float32)\n\n        # TODO: add species\n\n        return {\n            \"image\": img,\n            \"biomass\": biomass,\n            \"ndvi\": ndvi,\n            \"height\": height,\n            # \"species\": species\n        }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.566040Z","iopub.execute_input":"2025-12-27T23:44:59.566498Z","iopub.status.idle":"2025-12-27T23:44:59.580607Z","shell.execute_reply.started":"2025-12-27T23:44:59.566456Z","shell.execute_reply":"2025-12-27T23:44:59.579915Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class MultiHeadBiomassModel(nn.Module):\n    def __init__(self, model_path, freeze_backbone = True):\n        super().__init__()\n\n        self.backbone = AutoModel.from_pretrained(model_path, local_files_only = True)\n        \n        embed_dim = self.backbone.config.hidden_size\n\n        # freeze backbone parameters\n        if freeze_backbone:\n            for p in self.backbone.parameters():\n                p.requires_grad = False\n\n        # ---- heads ----\n        self.biomass_head = nn.Sequential(\n            nn.Linear(embed_dim, 256),\n            nn.ReLU(),\n            nn.Linear(256, 5)\n        )\n\n        self.ndvi_head = nn.Sequential(\n            nn.Linear(embed_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, 1)\n        )\n\n        self.height_head = nn.Sequential(\n            nn.Linear(embed_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, 1)\n        )\n\n        # TODO: species head (output size = num_species)\n        # self.species_head = # TODO\n\n    def forward(self, x):\n        outputs = self.backbone(x)\n        \n        z = outputs.last_hidden_state[:, 0]  # shape (B, D)\n\n        return {\n            \"biomass\": self.biomass_head(z),\n            \"ndvi\": self.ndvi_head(z).squeeze(1),\n            \"height\": self.height_head(z).squeeze(1),\n            # \"species\": # TODO\n        }\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.581461Z","iopub.execute_input":"2025-12-27T23:44:59.582038Z","iopub.status.idle":"2025-12-27T23:44:59.601981Z","shell.execute_reply.started":"2025-12-27T23:44:59.582005Z","shell.execute_reply":"2025-12-27T23:44:59.601392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compute_loss(outputs, batch, loss_weights):\n    mse = nn.MSELoss()\n\n    cross_entropy = nn.CrossEntropyLoss()\n\n    loss_biomass = mse(outputs[\"biomass\"], batch[\"biomass\"])\n\n    loss_ndvi = mse(outputs[\"ndvi\"], batch[\"ndvi\"])\n\n    loss_height = mse(outputs[\"height\"], batch[\"height\"])\n\n    # TODO: compute species loss\n    # loss_species = \n\n    # TODO: weighted sum\n    total_loss = loss_weights[\"biomass\"] * loss_biomass + loss_weights[\"ndvi\"] * loss_ndvi + loss_weights[\"height\"] * loss_height\n\n    return total_loss\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.603056Z","iopub.execute_input":"2025-12-27T23:44:59.603436Z","iopub.status.idle":"2025-12-27T23:44:59.623433Z","shell.execute_reply.started":"2025-12-27T23:44:59.603404Z","shell.execute_reply":"2025-12-27T23:44:59.622708Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_one_epoch(model, loader, optimizer, device, loss_weights):\n    model.train()\n\n    total_loss = 0\n\n    for batch in loader:\n        images = batch[\"image\"].to(device)\n\n        # move targets to device\n        for key in batch:\n            batch[key] = batch[key].to(device)\n\n        # TODO: forward pass\n        outputs = model(images)\n\n        # TODO: compute loss\n        loss = compute_loss(outputs, batch, loss_weights)\n\n        # TODO: optimizer step\n        optimizer.zero_grad()\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n    # TODO: return average loss\n    return total_loss / len(loader)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.624254Z","iopub.execute_input":"2025-12-27T23:44:59.624485Z","iopub.status.idle":"2025-12-27T23:44:59.638605Z","shell.execute_reply.started":"2025-12-27T23:44:59.624462Z","shell.execute_reply":"2025-12-27T23:44:59.637847Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def validate(model, loader, device):\n    model.eval()\n\n    preds = []\n    targets = []\n\n    with torch.no_grad():\n        for batch in loader:\n            out = model(batch[\"image\"].to(device))\n\n            # store predictions\n            preds.append(out[\"biomass\"].cpu())\n\n            # store ground truth\n            targets.append(batch[\"biomass\"].cpu())\n\n    # N x 5\n    preds = torch.cat(preds).numpy()\n\n    targets = torch.cat(targets).numpy()\n\n    r2_weights = np.array([0.1, 0.1, 0.1, 0.5, 0.2])\n\n    # compute weighted R^2\n    SSRes = np.sum((preds - targets) ** 2, axis = 0)\n    SSTot = np.sum((targets.mean(axis = 0) - targets) ** 2, axis = 0)\n\n    r2_per_target = 1 - SSRes / SSTot\n    \n    score = np.sum(r2_per_target * r2_weights)\n\n    return score\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.639558Z","iopub.execute_input":"2025-12-27T23:44:59.639825Z","iopub.status.idle":"2025-12-27T23:44:59.661669Z","shell.execute_reply.started":"2025-12-27T23:44:59.639796Z","shell.execute_reply":"2025-12-27T23:44:59.660914Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Cross Validation","metadata":{}},{"cell_type":"code","source":"def run_kfold_cv(df, transform, n_splits=5):\n    kf = KFold(n_splits, shuffle = True, random_state = 67)\n\n    losses = []\n    vals = []\n\n    for fold, (train_idx, val_idx) in enumerate(kf.split(df)):\n        print(f\"\\n===== Fold {fold} =====\")\n\n        # split df\n        train_df = df.iloc[train_idx]\n        val_df = df.iloc[val_idx]\n\n        # create datasets\n        train_ds = BiomassDataset(train_df, transform)\n        val_ds = BiomassDataset(val_df, transform)\n\n        # create dataloaders\n        train_loader = DataLoader(train_ds, batch_size = 8, shuffle=True)\n        val_loader = DataLoader(val_ds, batch_size = 8, shuffle=True)\n\n        # initialize model\n        model = MultiHeadBiomassModel(model_path = MODEL_PATH, freeze_backbone = True)\n        model = model.to(device)\n\n        optimizer = torch.optim.AdamW(\n            filter(lambda p: p.requires_grad, model.parameters()),\n            lr=1e-3 \n        )\n\n        loss_weights = {\n            \"biomass\": 1.0,\n            \"ndvi\": 0.2,\n            \"height\": 0.2,\n            # \"species\": 0.1\n        }\n\n        num_epochs = 15\n        \n        for epoch in range(num_epochs):\n            train_loss = train_one_epoch(model, train_loader, optimizer, device, loss_weights)\n            val_score = validate(model, val_loader, device)\n\n            print(f\"Epoch {epoch}: loss={train_loss:.4f}, R2={val_score:.4f}\")\n\n            # Save last loss and val score\n            if epoch == num_epochs - 1:\n                losses.append(train_loss)\n                vals.append(val_score)\n                \n    print(f\"Average Train Loss: {np.mean(np.array(losses))}\")\n    print(f\"Average Validation Score: {np.mean(np.array(vals))}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:44:59.662881Z","iopub.execute_input":"2025-12-27T23:44:59.663186Z","iopub.status.idle":"2025-12-27T23:44:59.678844Z","shell.execute_reply.started":"2025-12-27T23:44:59.663156Z","shell.execute_reply":"2025-12-27T23:44:59.678141Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transform = T.Compose([\n    T.Resize((256, 256)),\n    T.ToTensor(),\n    T.Normalize(\n        mean=(0.485, 0.456, 0.406),\n        std=(0.229, 0.224, 0.225),\n    ),\n])\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# run_kfold_cv(df_wide, transform, n_splits = 5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T23:45:09.267045Z","iopub.execute_input":"2025-12-27T23:45:09.267384Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Training Loop","metadata":{}},{"cell_type":"code","source":"full_ds = BiomassDataset(df_wide, transform)\nfull_loader = DataLoader(\n    full_ds,\n    batch_size=8,\n    shuffle=True\n)\n\nmodel = MultiHeadBiomassModel(\n    model_path = MODEL_PATH, freeze_backbone = True\n).to(device)\n\noptimizer = torch.optim.AdamW(\n    filter(lambda p: p.requires_grad, model.parameters()),\n    lr=1e-3\n)\n\nbest_loss = float(\"inf\")\nBEST_MODEL_PATH = \"/kaggle/working/best_model.pth\"\n\nnum_epochs = 20\n\nloss_weights = {\n            \"biomass\": 1.0,\n            \"ndvi\": 0.2,\n            \"height\": 0.2,\n            # \"species\": 0.1\n        }\n\nfor epoch in range(num_epochs):\n    loss = train_one_epoch(\n        model, full_loader, optimizer, device, loss_weights\n    )\n\n    print(f\"Epoch {epoch}: loss={loss:.4f}\")\n\n    # save best checkpoint\n    if loss < best_loss:\n        best_loss = loss\n        torch.save(\n            {\n                \"epoch\": epoch,\n                \"model_state_dict\": model.state_dict(),\n                \"optimizer_state_dict\": optimizer.state_dict(),\n                \"loss\": best_loss,\n                \"dino_path\": \"/kaggle/input/dinov2/pytorch/small/1\",\n            },\n            BEST_MODEL_PATH\n        )\n        print(f\"Saved new best model (loss={best_loss:.4f})\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T21:47:04.939924Z","iopub.execute_input":"2025-12-27T21:47:04.940197Z","iopub.status.idle":"2025-12-27T21:54:48.049146Z","shell.execute_reply.started":"2025-12-27T21:47:04.940168Z","shell.execute_reply":"2025-12-27T21:54:48.048298Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, df, transform):\n        # df should have one row per image\n        self.df = df.reset_index(drop=True)\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n\n        img = Image.open(f\"{DATA_PATH}/{row['image_path']}\").convert(\"RGB\")\n        img = self.transform(img)\n\n        return {\n            \"image\": img,\n            \"image_id\": row[\"image_id\"],  # base ID (no target suffix)\n        }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T21:54:48.235280Z","iopub.execute_input":"2025-12-27T21:54:48.235617Z","iopub.status.idle":"2025-12-27T21:54:48.247301Z","shell.execute_reply.started":"2025-12-27T21:54:48.235593Z","shell.execute_reply":"2025-12-27T21:54:48.246700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def create_submission():\n    # ---- load model ----\n    ckpt = torch.load(BEST_MODEL_PATH, map_location=device)\n\n    model = MultiHeadBiomassModel(\n        model_path=ckpt[\"dino_path\"],\n        freeze_backbone = True\n    ).to(device)\n\n    model.load_state_dict(ckpt[\"model_state_dict\"])\n    model.eval()\n\n    # ---- load test csv ----\n    test_long = pd.read_csv(f\"{DATA_PATH}/test.csv\")\n\n    # extract base image id from sample_id\n    test_long[\"image_id\"] = test_long[\"sample_id\"].str.split(\"__\").str[0]\n\n    # unique images only\n    test_images = (\n        test_long[[\"image_id\", \"image_path\"]]\n        .drop_duplicates()\n        .reset_index(drop=True)\n    )\n\n    # ---- dataset / loader ----\n    test_dataset = TestDataset(test_images, transform)\n    test_loader = DataLoader(test_dataset, batch_size=8, shuffle=False)\n\n    TARGET_NAMES = [\n        \"Dry_Clover_g\",\n        \"Dry_Dead_g\",\n        \"Dry_Green_g\",\n        \"Dry_Total_g\",\n        \"GDM_g\",\n    ]\n\n    rows = []\n\n    # ---- inference ----\n    with torch.no_grad():\n        for batch in test_loader:\n            images = batch[\"image\"].to(device)\n            image_ids = batch[\"image_id\"]\n\n            preds = model(images)[\"biomass\"]  # (B, 5)\n            preds = preds.cpu().numpy()\n\n            for img_id, pred in zip(image_ids, preds):\n                for name, value in zip(TARGET_NAMES, pred):\n                    rows.append({\n                        \"sample_id\": f\"{img_id}__{name}\",\n                        \"target\": float(value),\n                    })\n\n    # ---- build submission ----\n    submission = pd.DataFrame(rows)\n\n    SUB_PATH = \"submission.csv\"\n    submission.to_csv(SUB_PATH, index=False)\n\n    print(f\"Submission saved to {SUB_PATH}\")\n    return submission\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T22:00:59.539979Z","iopub.execute_input":"2025-12-27T22:00:59.540530Z","iopub.status.idle":"2025-12-27T22:00:59.547954Z","shell.execute_reply.started":"2025-12-27T22:00:59.540500Z","shell.execute_reply":"2025-12-27T22:00:59.547366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = create_submission()\nsubmission.head()\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-27T22:01:07.963999Z","iopub.execute_input":"2025-12-27T22:01:07.964666Z","iopub.status.idle":"2025-12-27T22:01:08.446023Z","shell.execute_reply.started":"2025-12-27T22:01:07.964618Z","shell.execute_reply":"2025-12-27T22:01:08.445266Z"}},"outputs":[],"execution_count":null}]}