{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"sourceType":"competition"},{"sourceId":13747888,"sourceType":"datasetVersion","datasetId":8748054},{"sourceId":14302290,"sourceType":"datasetVersion","datasetId":9129968}],"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Install FastAI\n\nInstall the FastAI library and its dependencies using pip.","metadata":{}},{"cell_type":"code","source":"# Environment flag\nimport os\nkaggle = os.path.exists('/kaggle/input/csiro-biomass/')","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:46:47.974066Z","iopub.execute_input":"2026-01-05T06:46:47.974780Z","iopub.status.idle":"2026-01-05T06:46:47.981400Z","shell.execute_reply.started":"2026-01-05T06:46:47.974753Z","shell.execute_reply":"2026-01-05T06:46:47.980646Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#!pip install fastai","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:46:47.983164Z","iopub.execute_input":"2026-01-05T06:46:47.983436Z","iopub.status.idle":"2026-01-05T06:46:47.998229Z","shell.execute_reply.started":"2026-01-05T06:46:47.983410Z","shell.execute_reply":"2026-01-05T06:46:47.997548Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# === Fastai 2.8.5 Kaggle-Safe Offline Install (One Cell) ===\nimport sys\n\nif os.path.exists(\"/kaggle\"):\n    # Copy fastai source to a writable directory\n    !cp -r /kaggle/input/fastai-2-8-5/fastai-2.8.5 /kaggle/working/fastai_src\n\n    # Use the local source directly (editable)\n    sys.path.insert(0, \"/kaggle/working/fastai_src\")\n\n    # Auto-reload edits\n    %load_ext autoreload\n    %autoreload 2\n\n    print(\"Fastai 2.8.5 ready for editable development on Kaggle.\")\nelse:\n    print(\"Not on Kaggle, skipping FastAI setup.\")\n","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:46:47.999129Z","iopub.execute_input":"2026-01-05T06:46:47.999585Z","iopub.status.idle":"2026-01-05T06:46:49.868419Z","shell.execute_reply.started":"2026-01-05T06:46:47.999555Z","shell.execute_reply":"2026-01-05T06:46:49.867606Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.vision.all import *\nfrom fastai.metrics import R2Score\nfrom fastai.tabular.all import *\nimport pandas as pd\nimport numpy as np\n\nimport os\nimport warnings\nfrom torchvision.models import ConvNeXt_Tiny_Weights, ConvNeXt_Base_Weights, ConvNeXt_Large_Weights\nfrom torchvision.models import convnext_tiny, convnext_large, convnext_base\n\n# Suppress torchvision warnings\nwarnings.filterwarnings(\"ignore\", category=UserWarning, module=\"torchvision\")\n\n# Set PyTorch CUDA memory management\nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:46:49.870473Z","iopub.execute_input":"2026-01-05T06:46:49.871072Z","iopub.status.idle":"2026-01-05T06:47:03.333420Z","shell.execute_reply.started":"2026-01-05T06:46:49.871044Z","shell.execute_reply":"2026-01-05T06:47:03.332861Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nimport fastai\nprint(fastai.__version__)\n\n# Example: check torch and GPU\nimport torch\nprint(torch.__version__, torch.cuda.is_available())","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:03.334085Z","iopub.execute_input":"2026-01-05T06:47:03.334263Z","iopub.status.idle":"2026-01-05T06:47:03.458568Z","shell.execute_reply.started":"2026-01-05T06:47:03.334248Z","shell.execute_reply":"2026-01-05T06:47:03.457787Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Import Libraries\n\nImport necessary libraries including FastAI's vision module.","metadata":{}},{"cell_type":"markdown","source":"# Load Dataset\n\nLoad the biomass dataset from the csiro-biomass folder.","metadata":{}},{"cell_type":"code","source":"# Pick data directory\ndata_dir = \"/kaggle/input/csiro-biomass/\" if kaggle else \"./csiro-biomass/\"\npath = Path(data_dir)\n\ndf = pd.read_csv(path/'train.csv')\n\n# Convert Sampling_Date to day of year for better use as feature\ndf['Sampling_Date'] = pd.to_datetime(df['Sampling_Date'])\ndf['DayOfYear'] = df['Sampling_Date'].dt.dayofyear\n\n# Define target columns\ntarget_cols = ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\n\n# Metadata columns\ncat_names = ['State', 'Species']\ncont_names = ['Pre_GSHH_NDVI', 'Height_Ave_cm', 'DayOfYear']\n\n# Pivot to have one row per image with all targets\ndf_pivot = df.pivot(index='image_path', columns='target_name', values='target').reset_index()\n\n# Get metadata per image\nmetadata_cols = cat_names + cont_names\nmetadata_df = df.groupby('image_path')[metadata_cols].first().reset_index()\n\n# Merge metadata with pivoted df\ndf_pivot = df_pivot.merge(metadata_df, on='image_path', how='left')\n\n# Make image_path full path\ndf_pivot['image_path'] = df_pivot['image_path'].apply(lambda x: str(path / x))\n\n# Add NDVI derivatives (safe for negative NDVI)\nndvi = df_pivot['Pre_GSHH_NDVI'].astype(float)\ndf_pivot['NDVI_squared'] = ndvi ** 2\ndf_pivot['NDVI_cubed'] = ndvi ** 3\n# Signed log transform: preserves sign and avoids NaNs for negative values\ndf_pivot['NDVI_log'] = np.sign(ndvi) * np.log1p(np.abs(ndvi))\ncont_names = cont_names + ['NDVI_squared', 'NDVI_cubed', 'NDVI_log']\n\n# Drop rows with missing targets (keep metadata NaNs if any)\ndf_pivot = df_pivot.dropna(subset=target_cols).reset_index(drop=True)\n\nprint(df_pivot.head())\nprint(f\"Dataset size after dropping NaN targets: {len(df_pivot)}\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:03.459479Z","iopub.execute_input":"2026-01-05T06:47:03.459791Z","iopub.status.idle":"2026-01-05T06:47:03.586417Z","shell.execute_reply.started":"2026-01-05T06:47:03.459766Z","shell.execute_reply":"2026-01-05T06:47:03.585888Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:03.587124Z","iopub.execute_input":"2026-01-05T06:47:03.587418Z","iopub.status.idle":"2026-01-05T06:47:03.648261Z","shell.execute_reply.started":"2026-01-05T06:47:03.587398Z","shell.execute_reply":"2026-01-05T06:47:03.647631Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Create DataLoaders\n\nCreate DataLoaders for training and validation using FastAI's data loading utilities.","metadata":{}},{"cell_type":"code","source":"# --- Leakage-safe split + train-only normalization ---\nseed = 42\ntrain_idxs, valid_idxs = RandomSplitter(seed=seed)(range_of(df_pivot))\ntrain_df = df_pivot.iloc[train_idxs]\n\n# Train-only stats\ncont_means = train_df[cont_names].mean()\ncont_stds = train_df[cont_names].std().replace(0, 1)\ntarget_means = train_df[target_cols].mean()\ntarget_stds = train_df[target_cols].std().replace(0, 1)\n\n# Train-only categorical vocab (unknown -> 0)\nstate_vocab = ['<UNK>'] + sorted(train_df['State'].dropna().unique().tolist())\nspecies_vocab = ['<UNK>'] + sorted(train_df['Species'].dropna().unique().tolist())\nstate2i = {v: i for i, v in enumerate(state_vocab)}\nspecies2i = {v: i for i, v in enumerate(species_vocab)}\n\ndefault_state = train_df['State'].mode().iloc[0] if train_df['State'].notna().any() else state_vocab[0]\ndefault_species = train_df['Species'].mode().iloc[0] if train_df['Species'].notna().any() else species_vocab[0]\n\n# Embedding sizes for TabularModel\nfrom fastai.tabular.model import emb_sz_rule\nemb_szs = [\n    (len(state_vocab), emb_sz_rule(len(state_vocab))),\n    (len(species_vocab), emb_sz_rule(len(species_vocab))),\n]\n\ndef _norm_cont(val, name):\n    m = float(cont_means[name])\n    s = float(cont_stds[name])\n    if pd.isna(val):\n        val = m\n    return (float(val) - m) / s\n\ndef encode_tabular(r):\n    state = r.get('State', default_state)\n    species = r.get('Species', default_species)\n    state_code = state2i.get(state, 0)\n    species_code = species2i.get(species, 0)\n\n    # IMPORTANT: cont features must match cont_names used for train-only stats and n_cont\n    cont_vals = [_norm_cont(r.get(name, np.nan), name) for name in cont_names]\n\n    return torch.tensor([state_code, species_code, *cont_vals], dtype=torch.float32)\n\ndef get_targets(r):\n    vals = r[target_cols].to_numpy(dtype=np.float32)\n    normed = (vals - target_means.to_numpy(dtype=np.float32)) / target_stds.to_numpy(dtype=np.float32)\n    return normed\n\ndblock = DataBlock(\n    blocks=(\n        ImageBlock,\n        TransformBlock(),\n        TransformBlock(type_tfms=[lambda x: tensor(x).float()])\n    ),\n    get_x=[ColReader('image_path'), encode_tabular],\n    get_y=get_targets,\n    item_tfms=Resize(192),\n    splitter=IndexSplitter(valid_idxs),\n)\n\nbs = 4\ndls = dblock.dataloaders(df_pivot, bs=bs)","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:03.648915Z","iopub.execute_input":"2026-01-05T06:47:03.649130Z","iopub.status.idle":"2026-01-05T06:47:04.162256Z","shell.execute_reply.started":"2026-01-05T06:47:03.649111Z","shell.execute_reply":"2026-01-05T06:47:04.161648Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show labeled random examples from each target\nimport random\nimport torch\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nnum_examples = 5\nindices = random.sample(range(len(dls.train_ds)), num_examples)\n\nfor idx in indices:\n    img, tab, targ = dls.train_ds[idx]\n    targ_orig = targ * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\n    \n    # Get additional metadata\n    img_path = dls.train_ds.items.iloc[idx]['image_path']\n    img_path_rel = Path(img_path).relative_to(path).as_posix()\n    row = df[df['image_path'] == img_path_rel].iloc[0]\n    species = row['Species']\n    state = row['State']\n    height = row['Height_Ave_cm']\n    \n    # Display image with label below\n    fig, ax = plt.subplots()\n    img_np = np.array(img)\n    ax.imshow(img_np)\n    ax.axis('off')\n    label = f\"Example {idx}: Species: {species}, State: {state}, Height: {height:.1f} cm\\nTargets: {dict(zip(target_cols, targ_orig.tolist()))}\"\n    plt.figtext(0.5, 0.01, label, ha='center', fontsize=8, wrap=True)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:04.162915Z","iopub.execute_input":"2026-01-05T06:47:04.163114Z","iopub.status.idle":"2026-01-05T06:47:06.383119Z","shell.execute_reply.started":"2026-01-05T06:47:04.163095Z","shell.execute_reply":"2026-01-05T06:47:06.382395Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport torch\nimport torch.nn as nn\nfrom fastai.vision.all import AdaptiveConcatPool2d, Learner, R2Score\n\n# ---------------------------\n# Multi-Modal Model (fastai-friendly device handling)\n# ---------------------------\nclass MultiModalModel(nn.Module):\n    def __init__(self, cnn_body, tabular_model, n_cat, n_cont, out_sz=5,\n                 head_layers=[1024, 512], dropout=[0.3, 0.2]):\n        super().__init__()\n        self.cnn_body = cnn_body\n        self.pool = AdaptiveConcatPool2d()\n        self.tabular_model = tabular_model\n        self.n_cat = n_cat\n        self.n_cont = n_cont\n\n        # Compute feature sizes once (on whatever device the submodules are currently on)\n        with torch.no_grad():\n            try:\n                cnn_dev = next(self.cnn_body.parameters()).device\n            except StopIteration:\n                cnn_dev = torch.device('cpu')\n            try:\n                tab_dev = next(self.tabular_model.parameters()).device\n            except StopIteration:\n                tab_dev = torch.device('cpu')\n\n            dummy_img = torch.randn(1, 3, 224, 224, device=cnn_dev)\n            cnn_out = self.pool(self.cnn_body(dummy_img)).flatten(1).shape[1]\n\n            dummy_cat = torch.zeros(2, n_cat, dtype=torch.long, device=tab_dev)\n            dummy_cont = torch.randn(2, n_cont, device=tab_dev)\n            self.tabular_model.eval()\n            tab_out = self.tabular_model(dummy_cat, dummy_cont).shape[1]\n            self.tabular_model.train()\n\n        layers = []\n        in_features = cnn_out + tab_out\n        for i, h in enumerate(head_layers):\n            layers.append(nn.Linear(in_features, h))\n            layers.append(nn.ReLU())\n            layers.append(nn.Dropout(dropout[i]))\n            in_features = h\n        layers.append(nn.Linear(in_features, out_sz))\n        self.head = nn.Sequential(*layers)\n\n    def forward(self, img, tab):\n        # fastai will place tensors on the right device; keep it simple\n        cat = tab[:, :self.n_cat].long()\n        cont = tab[:, self.n_cat:]\n        img_feat = self.pool(self.cnn_body(img)).flatten(1)\n        tab_feat = self.tabular_model(cat, cont)\n        combined = torch.cat([img_feat, tab_feat], dim=1)\n        return self.head(combined)\n\n\n# ---------------------------\n# Multi-Task Loss\n# ---------------------------\nclass MultiTaskLoss(nn.Module):\n    def __init__(self, weights=None, alpha=0.1, target_means=None, target_stds=None):\n        \"\"\"\n        Args:\n            weights: Tensor/array of shape [num_targets], boosts weak targets\n            alpha: weight for total-consistency loss\n            target_means/target_stds: train-only stats for denorm inside loss\n        \"\"\"\n        super().__init__()\n        if weights is None:\n            weights = torch.ones(5)\n        self.weights = torch.as_tensor(weights, dtype=torch.float32)\n        self.alpha = alpha\n        self.mse = nn.MSELoss(reduction='none')\n\n        means = torch.as_tensor(np.asarray(target_means), dtype=torch.float32) if target_means is not None else torch.zeros(5)\n        stds = torch.as_tensor(np.asarray(target_stds), dtype=torch.float32) if target_stds is not None else torch.ones(5)\n        self.register_buffer('tmean', means)\n        self.register_buffer('tstd', stds)\n\n    def forward(self, preds, targets):\n        # Weighted MSE in normalized space (stable optimization)\n        base_loss = (self.mse(preds, targets) * self.weights.to(preds.device)).mean()\n\n        # Consistency loss computed in original units (avoids per-target z-score mismatch)\n        preds_orig = preds * self.tstd.to(preds.device) + self.tmean.to(preds.device)\n        total_from_parts = preds_orig[:, 0] + preds_orig[:, 1] + preds_orig[:, 2]\n        total_pred = preds_orig[:, 4]\n        consistency_loss = nn.functional.mse_loss(total_from_parts, total_pred)\n\n        return base_loss + self.alpha * consistency_loss","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:06.385690Z","iopub.execute_input":"2026-01-05T06:47:06.386045Z","iopub.status.idle":"2026-01-05T06:47:06.442877Z","shell.execute_reply.started":"2026-01-05T06:47:06.386029Z","shell.execute_reply":"2026-01-05T06:47:06.442300Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------------\n# 1️⃣ Create tabular model\n# ---------------------------\nn_cont = len(cont_names)\nn_cat = len(cat_names)  # number of categorical columns (State, Species)\ntabular_model = TabularModel(emb_szs, n_cont, layers=[200,100], out_sz=50)","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:06.443940Z","iopub.execute_input":"2026-01-05T06:47:06.444183Z","iopub.status.idle":"2026-01-05T06:47:06.500816Z","shell.execute_reply.started":"2026-01-05T06:47:06.444155Z","shell.execute_reply":"2026-01-05T06:47:06.500271Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"if kaggle:\n    # Load custom pretrained weights from CSIRO ConvNeXt Base\n    model = convnext_base(weights=None)\n    weights_path = \"/kaggle/input/convnext-base-weights/convnext_base-6075fbad.pth\"\n    model.load_state_dict(torch.load(weights_path))\n    print(\"Using ConvNeXt base with CSIRO pretrained weights on Kaggle\")\nelse:\n    model = convnext_base(weights=None)\n    weights_path = \"weights/convnext_base-6075fbad.pth\"\n    if os.path.exists(weights_path):\n        model.load_state_dict(torch.load(weights_path))\n        print(\"Using ConvNeXt base with local pretrained weights\")\n    else:\n        print(\"Using ConvNeXt base with random initialization locally\")\n\n# Create CNN body\ncnn_body = create_body(model, pretrained=False)","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:06.501460Z","iopub.execute_input":"2026-01-05T06:47:06.501720Z","iopub.status.idle":"2026-01-05T06:47:10.095687Z","shell.execute_reply.started":"2026-01-05T06:47:06.501698Z","shell.execute_reply":"2026-01-05T06:47:10.094920Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ---------------------------\n# 2️⃣ Create multi-modal model\n# ---------------------------\nmodel = MultiModalModel(cnn_body, tabular_model, n_cat=n_cat, n_cont=n_cont, out_sz=5)\n\n# ---------------------------\n# 3️⃣ Define multi-task loss (consistency computed in original units)\n# ---------------------------\n# Order matches target_cols: ['Dry_Green_g', 'Dry_Dead_g', 'Dry_Clover_g', 'GDM_g', 'Dry_Total_g']\nloss_weights = torch.tensor([1.0, 1.5, 1.5, 1.0, 1.0])\nloss_func = MultiTaskLoss(weights=loss_weights, alpha=0.1, target_means=target_means, target_stds=target_stds)\n\n# ---------------------------\n# 4️⃣ Create Learner\n# ---------------------------\nlearn = Learner(dls, model, loss_func=loss_func, metrics=[R2Score()])","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:10.096553Z","iopub.execute_input":"2026-01-05T06:47:10.096881Z","iopub.status.idle":"2026-01-05T06:47:10.493232Z","shell.execute_reply.started":"2026-01-05T06:47:10.096862Z","shell.execute_reply":"2026-01-05T06:47:10.492462Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train Model\n\nInitialize a learner with a pre-trained model and fine-tune it on the dataset.","metadata":{}},{"cell_type":"code","source":"# ---------------------------\n# 1️⃣ Ensure batch size >= 2\n# ---------------------------\ndls.train.bs = max(dls.train.bs, 2)\ndls.valid.bs = max(dls.valid.bs, 2)\n\n# ---------------------------\n# 2️⃣ LR Finder in FP32\n# ---------------------------\n# Do NOT call learn.to_fp16() yet\nlr_min, lr_steep = learn.lr_find(suggest_funcs=(minimum, steep))\nprint(f\"Safe LR (valley): {lr_min:.2e}, Steepest LR: {lr_steep:.2e}\")\n","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:10.494031Z","iopub.execute_input":"2026-01-05T06:47:10.494243Z","iopub.status.idle":"2026-01-05T06:47:24.938326Z","shell.execute_reply.started":"2026-01-05T06:47:10.494227Z","shell.execute_reply":"2026-01-05T06:47:24.937558Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\n\n# Convenience: show device fastai will use\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(device)  # 'cuda' if GPU is available","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:24.939350Z","iopub.execute_input":"2026-01-05T06:47:24.939619Z","iopub.status.idle":"2026-01-05T06:47:24.995837Z","shell.execute_reply.started":"2026-01-05T06:47:24.939597Z","shell.execute_reply":"2026-01-05T06:47:24.995068Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Define a splitter function\ndef multimodal_splitter(model):\n    # Returns list of parameter groups\n    return [\n        list(model.cnn_body.parameters()),     # group 1: CNN backbone\n        list(model.tabular_model.parameters()),# group 2: tabular\n        list(model.head.parameters())          # group 3: head\n    ]","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:24.996709Z","iopub.execute_input":"2026-01-05T06:47:24.997091Z","iopub.status.idle":"2026-01-05T06:47:25.047918Z","shell.execute_reply.started":"2026-01-05T06:47:24.997066Z","shell.execute_reply":"2026-01-05T06:47:25.047222Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from fastai.callback.all import SaveModelCallback, EarlyStoppingCallback\nfrom fastai.callback.schedule import minimum, steep\nimport torch\ntorch.cuda.empty_cache()\n\n# ---------------------------\n# 1️⃣ Manually convert tabular to FP32 and rest to FP16\n# ---------------------------\nlearn.model.tabular_model.float()   # ensure tabular stays in FP32\nlearn = learn.to_fp16()             # rest of model to FP16\n\n# ---------------------------\n# 2️⃣ Freeze backbone first (train head only)\n# ---------------------------\nlearn.freeze()\n\n# ---------------------------\n# 3️⃣ Find recommended learning rates\n# ---------------------------\nlr_min, lr_steep = learn.lr_find(suggest_funcs=(minimum, steep))\nprint(f\"Safe LR (valley/minimum): {lr_min:.2e}\")\nprint(f\"Steepest LR: {lr_steep:.2e}\")\n\n# ---------------------------\n# 4️⃣ Train head first\n# ---------------------------\nlearn.fit_one_cycle(5, lr_max=lr_min)\n\n# ---------------------------\n# 5️⃣ Unfreeze all layers for fine-tuning\n# ---------------------------\nlearn.unfreeze()\n\n# ---------------------------\n# 6️⃣ Split model into parameter groups for discriminative LRs\n# ---------------------------\n# Re-create learner with the splitter\nlearn = Learner(\n    dls, model,\n    loss_func=loss_func,\n    metrics=[R2Score()],\n    splitter=multimodal_splitter\n)\n\n# ---------------------------\n# 7️⃣ Define discriminative learning rates\n# ---------------------------\nlr_cnn  = lr_min / 10\nlr_tab  = lr_min / 5\nlr_head = lr_min\n\n# ---------------------------\n# 8️⃣ Callbacks\n# ---------------------------\ncallbacks = [\n    SaveModelCallback(monitor='r2_score', fname='best_model'),\n    EarlyStoppingCallback(monitor='r2_score', patience=10)\n]\nlr_max=[lr_cnn, lr_tab, lr_head]\n# ---------------------------\n# 9️⃣ Fine-tune full model with discriminative LRs\n# ---------------------------\nlearn.fit_one_cycle(\n    50, \n    lr_max=[lr_cnn, lr_tab, lr_head],\n    cbs=callbacks\n)\n","metadata":{"execution":{"iopub.status.busy":"2026-01-05T06:47:25.048730Z","iopub.execute_input":"2026-01-05T06:47:25.049002Z","iopub.status.idle":"2026-01-05T07:02:13.232222Z","shell.execute_reply.started":"2026-01-05T06:47:25.048986Z","shell.execute_reply":"2026-01-05T07:02:13.231459Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Plot training loss\nlearn.recorder.plot_loss()","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:13.233426Z","iopub.execute_input":"2026-01-05T07:02:13.233752Z","iopub.status.idle":"2026-01-05T07:02:13.515795Z","shell.execute_reply.started":"2026-01-05T07:02:13.233722Z","shell.execute_reply":"2026-01-05T07:02:13.515113Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Detailed Training Loss Plot\nplt.figure(figsize=(10, 6))\nplt.plot(learn.recorder.lrs, learn.recorder.losses, label='Training Loss')\nif hasattr(learn.recorder, 'val_losses') and learn.recorder.val_losses:\n    plt.plot(learn.recorder.lrs, learn.recorder.val_losses, label='Validation Loss')\nplt.xlabel('Learning Rate')\nplt.ylabel('Loss')\nplt.title('Training and Validation Loss vs Learning Rate')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:13.516669Z","iopub.execute_input":"2026-01-05T07:02:13.516963Z","iopub.status.idle":"2026-01-05T07:02:13.758951Z","shell.execute_reply.started":"2026-01-05T07:02:13.516937Z","shell.execute_reply":"2026-01-05T07:02:13.758304Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Evaluate Model\n\nEvaluate the model's performance using metrics like R2.","metadata":{}},{"cell_type":"code","source":"# For regression, plot_top_losses is not implemented. Instead, let's validate the model.\nlearn.validate()","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:13.759850Z","iopub.execute_input":"2026-01-05T07:02:13.760154Z","iopub.status.idle":"2026-01-05T07:02:15.901974Z","shell.execute_reply.started":"2026-01-05T07:02:13.760127Z","shell.execute_reply":"2026-01-05T07:02:15.901299Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Visualize Predictions\n\nSince this is a regression task, we'll create a scatter plot of predicted vs. actual biomass values and show examples of good and bad predictions.","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport torch\n\n# Get predictions on validation set\npreds, targs = learn.get_preds()\n\n# Denormalize predictions and targets\ntargs_orig = targs * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\npreds_orig = preds * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\n\n# Plot for each target\nfig, axes = plt.subplots(1, 5, figsize=(25, 5))\nfor i, target in enumerate(target_cols):\n    axes[i].scatter(targs_orig[:, i], preds_orig[:, i], alpha=0.5)\n    axes[i].plot([targs_orig[:, i].min(), targs_orig[:, i].max()], [targs_orig[:, i].min(), targs_orig[:, i].max()], 'r--')\n    axes[i].set_xlabel(f'Actual {target} (g)')\n    axes[i].set_ylabel(f'Predicted {target} (g)')\n    axes[i].set_title(f'Predicted vs Actual {target}')\nplt.tight_layout()\nplt.show()\n\n# Calculate errors and show best/worst for each target\nfor i, target in enumerate(target_cols):\n    errors = (preds_orig[:, i] - targs_orig[:, i]).abs()\n    \n    # Get indices of best and worst predictions\n    best_idx = errors.argsort()[:3]  # Top 3 best\n    worst_idx = errors.argsort(descending=True)[:3]  # Top 3 worst\n    \n    print(f\"\\nBest Predictions for {target}:\")\n    for j in best_idx:\n        idx = j.item()\n        img_path = dls.valid_ds.items.iloc[idx]['image_path']\n        actual = targs_orig[idx, i].item()\n        pred = preds_orig[idx, i].item()\n        print(f\"Image: {img_path}, Actual: {actual:.2f}, Predicted: {pred:.2f}, Error: {abs(pred - actual):.2f}\")\n    \n    print(f\"\\nWorst Predictions for {target}:\")\n    for j in worst_idx:\n        idx = j.item()\n        img_path = dls.valid_ds.items.iloc[idx]['image_path']\n        actual = targs_orig[idx, i].item()\n        pred = preds_orig[idx, i].item()\n        print(f\"Image: {img_path}, Actual: {actual:.2f}, Predicted: {pred:.2f}, Error: {abs(pred - actual):.2f}\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:15.902974Z","iopub.execute_input":"2026-01-05T07:02:15.903200Z","iopub.status.idle":"2026-01-05T07:02:18.845476Z","shell.execute_reply.started":"2026-01-05T07:02:15.903180Z","shell.execute_reply":"2026-01-05T07:02:18.844655Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Confusion Matrix equivalent for regression: Correlation Matrix\nimport seaborn as sns\n\n# Compute correlation matrix between actual and predicted\ncorr_matrix = np.corrcoef(targs_orig.T, preds_orig.T)[:5, 5:]\ncorr_df = pd.DataFrame(corr_matrix, index=target_cols, columns=[f'Pred_{col}' for col in target_cols])\n\nplt.figure(figsize=(8, 6))\nsns.heatmap(corr_df, annot=True, cmap='coolwarm', vmin=-1, vmax=1)\nplt.title('Correlation between Actual and Predicted Targets')\nplt.show()\n\nprint(\"Correlation Matrix:\")\nprint(corr_df)","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:18.846530Z","iopub.execute_input":"2026-01-05T07:02:18.847101Z","iopub.status.idle":"2026-01-05T07:02:19.385774Z","shell.execute_reply.started":"2026-01-05T07:02:18.847075Z","shell.execute_reply":"2026-01-05T07:02:19.385027Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(f\"preds.shape: {preds.shape}\")\nprint(f\"targs.shape: {targs.shape}\")\nprint(f\"target_means: {target_means}\")\nprint(f\"target_stds: {target_stds}\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:19.386593Z","iopub.execute_input":"2026-01-05T07:02:19.387044Z","iopub.status.idle":"2026-01-05T07:02:19.436487Z","shell.execute_reply.started":"2026-01-05T07:02:19.387019Z","shell.execute_reply":"2026-01-05T07:02:19.435708Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate estimated leaderboard score (weighted R² on validation set)\nimport torch\n\n# Denormalize\ntargs_orig = targs * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\npreds_orig = preds * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\n\n# Weights for each target\nweights_dict = {'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1, 'GDM_g': 0.2, 'Dry_Total_g': 0.5}\n\n# Compute R² for each target and weight them\nweighted_r2 = 0.0\nfor i, target in enumerate(target_cols):\n    targs_i = targs_orig[:, i]\n    preds_i = preds_orig[:, i]\n    mean_i = targs_i.mean()\n    SS_res = ((targs_i - preds_i)**2).sum()\n    SS_tot = ((targs_i - mean_i)**2).sum()\n    r2_i = 1 - SS_res / SS_tot\n    weight = weights_dict[target]\n    weighted_r2 += weight * r2_i\n    print(f\"{target}: R² = {r2_i:.4f}, weight = {weight}\")\n\nprint(f\"\\nEstimated Leaderboard Score (Weighted R²): {weighted_r2:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:19.437525Z","iopub.execute_input":"2026-01-05T07:02:19.437809Z","iopub.status.idle":"2026-01-05T07:02:19.496324Z","shell.execute_reply.started":"2026-01-05T07:02:19.437782Z","shell.execute_reply":"2026-01-05T07:02:19.495693Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pathlib\nimport pandas as pd\nimport torch\n\ntest_df = pd.read_csv(path/'test.csv')\ntest_df['image_path'] = test_df['image_path'].apply(lambda x: str(pathlib.Path(path) / x))\n\n# One row per unique image for prediction\ntest_unique = test_df[['image_path']].drop_duplicates().reset_index(drop=True)\n\n# Create dummy metadata for test set using *train-only* defaults/means\ntest_pivot = test_unique.copy()\ntest_pivot['State'] = default_state\ntest_pivot['Species'] = default_species\n\n# Populate ALL continuous features expected by the model\nfor name in cont_names:\n    if name == 'DayOfYear':\n        test_pivot[name] = int(round(float(cont_means[name])))\n    else:\n        test_pivot[name] = float(cont_means[name])\n\nprint(f\"Test DF size (rows): {test_df.shape}\")\nprint(f\"Unique images: {test_pivot.shape}\")\n\ntest_dl = dls.test_dl(test_pivot)\ntest_preds, _ = learn.get_preds(dl=test_dl)\n\n# Denormalize predictions (use train-only target stats)\ntmean = torch.tensor(target_means.to_numpy(dtype=np.float32), device=test_preds.device)\ntstd = torch.tensor(target_stds.to_numpy(dtype=np.float32), device=test_preds.device)\ntest_preds_orig = test_preds * tstd + tmean\n\n# Ensure non-negative predictions (biomass can't be negative)\ntest_preds_orig = torch.clamp(test_preds_orig, min=0)\n\n# Wide preds -> long preds keyed by image_path + target_name\npred_wide = pd.DataFrame(test_preds_orig.cpu().numpy(), columns=target_cols)\npred_wide['image_path'] = test_pivot['image_path'].values\npred_long = pred_wide.melt(id_vars='image_path', var_name='target_name', value_name='target')\n\n# Build submission by joining back to the original test rows\nsubmission = (\n    test_df[['sample_id', 'image_path', 'target_name']]\n    .merge(pred_long, on=['image_path', 'target_name'], how='left')\n    [['sample_id', 'target']]\n)\n\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission saved to submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:19.497054Z","iopub.execute_input":"2026-01-05T07:02:19.497285Z","iopub.status.idle":"2026-01-05T07:02:19.900082Z","shell.execute_reply.started":"2026-01-05T07:02:19.497270Z","shell.execute_reply":"2026-01-05T07:02:19.899425Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.head(10)","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:19.900944Z","iopub.execute_input":"2026-01-05T07:02:19.901215Z","iopub.status.idle":"2026-01-05T07:02:19.961017Z","shell.execute_reply.started":"2026-01-05T07:02:19.901192Z","shell.execute_reply":"2026-01-05T07:02:19.960353Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculate estimated leaderboard score (weighted R² on validation set)\nimport torch\n\n# Denormalize\ntargs_orig = targs * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\npreds_orig = preds * torch.tensor(target_stds.values) + torch.tensor(target_means.values)\n\n# Flatten all predictions and targets\nall_targs = targs_orig.flatten()\nall_preds = preds_orig.flatten()\n\n# Weights\nn_samples = targs.shape[0]\nweights_dict = {'Dry_Green_g': 0.1, 'Dry_Dead_g': 0.1, 'Dry_Clover_g': 0.1, 'GDM_g': 0.2, 'Dry_Total_g': 0.5}\n\nlb_weights = []\nfor target in target_cols:\n    lb_weights.extend([weights_dict[target]] * n_samples)\nlb_weights = torch.tensor(lb_weights)\n\n# Global weighted mean\nglobal_mean = (lb_weights * all_targs).sum() / lb_weights.sum()\n\n# SS_res and SS_tot\nSS_res = (lb_weights * (all_targs - all_preds)**2).sum()\nSS_tot = (lb_weights * (all_targs - global_mean)**2).sum()\n\nr2 = 1 - SS_res / SS_tot\nprint(f\"Estimated Leaderboard Score (Weighted R²): {r2:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2026-01-05T07:02:19.961835Z","iopub.execute_input":"2026-01-05T07:02:19.962095Z","iopub.status.idle":"2026-01-05T07:02:20.012202Z","shell.execute_reply.started":"2026-01-05T07:02:19.962072Z","shell.execute_reply":"2026-01-05T07:02:20.011392Z"},"trusted":true},"outputs":[],"execution_count":null}]}