{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.13"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":112509,"databundleVersionId":14254895,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":4537,"sourceType":"modelInstanceVersion","isSourceIdPinned":true,"modelInstanceId":3329,"modelId":986}],"dockerImageVersionId":31194,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageFilter\nimport os\nimport math\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import r2_score\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom torch.utils.data import TensorDataset, DataLoader\nimport tensorflow as tf\nimport torchvision.transforms as T\nfrom transformers import AutoModel, AutoImageProcessor\nimport warnings\n\nnp.random.seed(9)\n\nwarnings.filterwarnings(\"ignore\", message=\".*use_fast.*\")\nwarnings.filterwarnings(\"ignore\", message=\".*UnsupportedFieldAttributeWarning.*\")\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nif torch.cuda.is_available():\n    torch.backends.cudnn.benchmark = True","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2025-11-19T12:22:46.605579Z","iopub.execute_input":"2025-11-19T12:22:46.605832Z","iopub.status.idle":"2025-11-19T12:23:18.534832Z","shell.execute_reply.started":"2025-11-19T12:22:46.6058Z","shell.execute_reply":"2025-11-19T12:23:18.534034Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/csiro-biomass/train.csv')\ntrain = train.drop(columns = ['Sampling_Date','State','Species','Pre_GSHH_NDVI','Height_Ave_cm'])\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:23:18.536649Z","iopub.execute_input":"2025-11-19T12:23:18.537448Z","iopub.status.idle":"2025-11-19T12:23:18.579178Z","shell.execute_reply.started":"2025-11-19T12:23:18.537427Z","shell.execute_reply":"2025-11-19T12:23:18.578373Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/csiro-biomass/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:23:18.579875Z","iopub.execute_input":"2025-11-19T12:23:18.580128Z","iopub.status.idle":"2025-11-19T12:23:18.591704Z","shell.execute_reply.started":"2025-11-19T12:23:18.580102Z","shell.execute_reply":"2025-11-19T12:23:18.591065Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:23:18.59257Z","iopub.execute_input":"2025-11-19T12:23:18.592844Z","iopub.status.idle":"2025-11-19T12:23:18.600912Z","shell.execute_reply.started":"2025-11-19T12:23:18.592826Z","shell.execute_reply":"2025-11-19T12:23:18.600307Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def list_images(folder, exts=('.jpg','.jpeg','.png','.tif','.tiff')):\n    return sorted([os.path.join(folder, f) for f in os.listdir(folder) if f.lower().endswith(exts)])\n\ndef preprocess_batch(paths, target_size=(224,224)):\n    arrs=[]\n    for p in paths:\n        im = Image.open(p).convert(\"RGB\").resize(target_size)\n        a = np.asarray(im).astype(\"float32\")/255\n        arrs.append(a)\n    return np.stack(arrs, axis=0)\n\n# Edge detection helper: prefers OpenCV Canny; falls back to PIL FIND_EDGES\n\ndef edge_detect_image(im):\n    try:\n        import cv2\n        arr = np.array(im.convert(\"RGB\"))\n        gray = cv2.cvtColor(arr, cv2.COLOR_RGB2GRAY)\n        edges = cv2.Canny(gray, 100, 200)\n        edges_rgb = cv2.cvtColor(edges, cv2.COLOR_GRAY2RGB)\n        return Image.fromarray(edges_rgb)\n    except Exception:\n        # Fallback to PIL edge filter\n        return im.convert(\"RGB\").filter(ImageFilter.FIND_EDGES)\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:23:18.601611Z","iopub.execute_input":"2025-11-19T12:23:18.601929Z","iopub.status.idle":"2025-11-19T12:23:18.6158Z","shell.execute_reply.started":"2025-11-19T12:23:18.60191Z","shell.execute_reply":"2025-11-19T12:23:18.614977Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DINOV2_DIR = \"/kaggle/input/dinov2/pytorch/giant/1\"\nIMAGES_DIR = \"/kaggle/input/csiro-biomass/train\"\nBATCH = 32\n\ndino_processor = AutoImageProcessor.from_pretrained(DINOV2_DIR, local_files_only=True, use_fast=True)\ndino_model = AutoModel.from_pretrained(DINOV2_DIR, local_files_only=True)\ndino_model.eval().to(device)\n\npaths = list_images(IMAGES_DIR)\nembs = []\nnames = []\n\nfor i in range(0, len(paths), BATCH):\n    batch_paths = paths[i:i+BATCH]\n    imgs = []\n    for p in batch_paths:\n        im = Image.open(p).convert(\"RGB\")\n        imgs.append(im)\n    # Edge-detected versions (RGB)\n    edges = [edge_detect_image(im) for im in imgs]\n\n    # Prepare processor inputs for both original and edge images\n    inputs_orig = dino_processor(images=imgs, return_tensors=\"pt\")\n    inputs_edge = dino_processor(images=edges, return_tensors=\"pt\")\n    pixel_values_orig = inputs_orig[\"pixel_values\"].to(device)\n    pixel_values_edge = inputs_edge[\"pixel_values\"].to(device)\n\n    with torch.no_grad():\n        with torch.cuda.amp.autocast(enabled=(device.type=='cuda')):\n            outputs_o = dino_model(pixel_values=pixel_values_orig)\n            feats_o = getattr(outputs_o, \"pooler_output\", None)\n            if feats_o is None:\n                feats_o = outputs_o.last_hidden_state[:, 0]\n\n            outputs_e = dino_model(pixel_values=pixel_values_edge)\n            feats_e = getattr(outputs_e, \"pooler_output\", None)\n            if feats_e is None:\n                feats_e = outputs_e.last_hidden_state[:, 0]\n        feats_o = feats_o.detach().cpu().numpy()\n        feats_e = feats_e.detach().cpu().numpy()\n\n    # Concatenate original + edge embeddings\n    feats = np.concatenate([feats_o, feats_e], axis=1)\n    embs.append(feats)\n    names.extend([os.path.basename(p) for p in batch_paths])\n\nembs = np.concatenate(embs, axis=0)\n# Build combined column names\nhalf = embs.shape[1] // 2\ncols = [f\"emb_{i+1}\" for i in range(half)] + [f\"emb_edge_{i+1}\" for i in range(half)]\ntrain_emb = pd.DataFrame(embs, columns=cols)\ntrain_emb.insert(0, \"filename\", names)\ntrain_emb.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:23:18.616621Z","iopub.execute_input":"2025-11-19T12:23:18.616802Z","iopub.status.idle":"2025-11-19T12:25:38.07222Z","shell.execute_reply.started":"2025-11-19T12:23:18.616787Z","shell.execute_reply":"2025-11-19T12:25:38.071382Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train['image_filename'] = train['image_path'].map(lambda p: p.split('/')[-1])\ntrain = train.merge(train_emb, left_on='image_filename', right_on='filename', how='left')\ntrain.drop(columns=['image_filename', 'filename'], inplace=True)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:25:38.074797Z","iopub.execute_input":"2025-11-19T12:25:38.07504Z","iopub.status.idle":"2025-11-19T12:25:38.161037Z","shell.execute_reply.started":"2025-11-19T12:25:38.075019Z","shell.execute_reply":"2025-11-19T12:25:38.160366Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = pd.get_dummies(train, columns=['target_name'], drop_first=False)\ntrain = train.drop(columns=['sample_id','image_path','target_name'], errors='ignore')\ntrain.head()\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:25:38.161777Z","iopub.execute_input":"2025-11-19T12:25:38.162009Z","iopub.status.idle":"2025-11-19T12:25:38.203681Z","shell.execute_reply.started":"2025-11-19T12:25:38.16199Z","shell.execute_reply":"2025-11-19T12:25:38.20298Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X=train.drop(columns=['target'])\ny=train['target']","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:25:38.204474Z","iopub.execute_input":"2025-11-19T12:25:38.204787Z","iopub.status.idle":"2025-11-19T12:25:38.213116Z","shell.execute_reply.started":"2025-11-19T12:25:38.204769Z","shell.execute_reply":"2025-11-19T12:25:38.212449Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n_total = len(X)\nn_test = int(round(0.2 * n_total))\nn_test -= n_test % 5\nif n_test == 0:\n    n_test = 5\nn_train = n_total - n_test\nn_train -= n_train % 5\nif n_train == 0:\n    n_train = 5\n    n_test = n_total - n_train\nX_train = X.iloc[:n_train].reset_index(drop=True)\ny_train = y.iloc[:n_train].reset_index(drop=True)\nX_test = X.iloc[n_train:n_train + n_test].reset_index(drop=True)\ny_test = y.iloc[n_train:n_train + n_test].reset_index(drop=True)\nprint(f\"Train/Test sizes (no shuffle): {len(X_train)}/{len(X_test)}\")","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:25:38.214015Z","iopub.execute_input":"2025-11-19T12:25:38.21435Z","iopub.status.idle":"2025-11-19T12:25:38.233204Z","shell.execute_reply.started":"2025-11-19T12:25:38.214322Z","shell.execute_reply":"2025-11-19T12:25:38.232164Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class_weights = {\n    'Dry_Green_g': 0.1,\n    'Dry_Dead_g': 0.1,\n    'Dry_Clover_g': 0.1,\n    'GDM_g': 0.2,\n    'Dry_Total_g': 0.5\n}\n\nonehot_cols = [c for c in X_train.columns if c.startswith(\"target_name_\")]\n\ndef infer_class(row):\n    for col in onehot_cols:\n        if row[col] == 1:\n            return col.replace(\"target_name_\", \"\")\n    return None\n\ndef weighted_r2(preds, labels, weights):\n    w = weights.astype(np.float64)\n    y = labels.astype(np.float64)\n    p = preds.astype(np.float64).flatten()\n    wmean = np.sum(w * y) / np.sum(w)\n    ss_res = np.sum(w * (y - p) ** 2)\n    ss_tot = np.sum(w * (y - wmean) ** 2)\n    if ss_tot == 0:\n        return 0.0\n    r2 = 1.0 - ss_res / ss_tot\n    return max(r2, -1.0)\n\ndef r2_by_class(preds, labels, X_df):\n    classes = X_df.apply(infer_class, axis=1)\n    results = {}\n    for class_name in class_weights.keys():\n        mask = classes == class_name\n        if mask.sum() == 0:\n            continue\n        y_class = labels[mask]\n        p_class = preds[mask]\n        w_class = np.full(mask.sum(), class_weights[class_name])\n        r2_val = weighted_r2(p_class, y_class, w_class)\n        results[class_name] = {\n            'r2': r2_val,\n            'count': int(mask.sum()),\n            'weight': class_weights[class_name]\n        }\n    return results\n\nn_total_tr = len(X_train)\nn_val = int(round(0.2 * n_total_tr))\nn_val -= n_val % 5\nif n_val == 0:\n    n_val = 5\nn_tr = n_total_tr - n_val\nn_tr -= n_tr % 5\nif n_tr == 0:\n    n_tr = 5\n    n_val = n_total_tr - n_tr\nX_tr = X_train.iloc[:n_tr].reset_index(drop=True)\ny_tr = y_train.iloc[:n_tr].reset_index(drop=True)\nX_val = X_train.iloc[n_tr:n_tr + n_val].reset_index(drop=True)\ny_val = y_train.iloc[n_tr:n_tr + n_val].reset_index(drop=True)\nprint(f\"Train/Val sizes (no shuffle): {len(X_tr)}/{len(X_val)}\")\n\nemb_cols_tr = [c for c in X_tr.columns if c.startswith('emb_')]\n\nif len(emb_cols_tr) > 0:\n    arr = X_tr[emb_cols_tr].to_numpy().astype(np.float32)\n    norms = np.linalg.norm(arr, axis=1, keepdims=True) + 1e-8\n    X_tr[emb_cols_tr] = (arr / norms).astype(np.float32)\n    arrv = X_val[emb_cols_tr].to_numpy().astype(np.float32)\n    normsv = np.linalg.norm(arrv, axis=1, keepdims=True) + 1e-8\n    X_val[emb_cols_tr] = (arrv / normsv).astype(np.float32)\n\nX_min = X_tr.min(axis=0).to_numpy().astype(np.float32)\nX_max = X_tr.max(axis=0).to_numpy().astype(np.float32)\nrange_safe = np.where((X_max - X_min) < 1e-12, 1e-12, X_max - X_min)\n\n# Log-transform target then min-max scale in log space\ny_tr_log = np.log1p(y_tr.to_numpy().astype(np.float64))\ny_val_log = np.log1p(y_val.to_numpy().astype(np.float64))\ny_min = y_tr_log.min()\ny_max = y_tr_log.max()\ny_range = y_max - y_min if (y_max - y_min) != 0 else 1e-12\n\nfeature_columns = X_tr.columns.tolist()\n\ndef scale_features(df):\n    arr = df[feature_columns].to_numpy().astype(np.float32)\n    scaled = (arr - X_min) / range_safe\n    scaled = np.clip(scaled, 0.0, 1.0)\n    return pd.DataFrame(scaled, columns=feature_columns)\n\ndef scale_target(series):\n    log_arr = np.log1p(series.to_numpy().astype(np.float64))\n    scaled = (log_arr - y_min) / y_range\n    return scaled.astype(np.float32)\n\ndef inverse_target(arr):\n    log_val = arr.astype(np.float64) * y_range + y_min\n    return np.expm1(log_val)\n\nbase_X_for_aug = X_tr.copy()\nif len(emb_cols_tr) > 0:\n    k = max(1, int(0.10 * len(emb_cols_tr)))\n    rounds_mask = 0\n    rounds_noise = 5\n    rng = np.random.default_rng(9)\n    total_added = 0\n    X_parts = [X_tr]\n    y_parts = [y_tr]\n    # mask augmentation disabled (rounds_mask = 0)\n    col_std = base_X_for_aug[emb_cols_tr].to_numpy().astype(np.float64).std(axis=0)\n    col_std[col_std == 0] = 1e-6\n    sigma = 0.10 * col_std\n    for _ in range(rounds_noise):\n        aug_rows = base_X_for_aug.copy()\n        emb_values = aug_rows[emb_cols_tr].to_numpy().astype(np.float64)\n        for i in range(len(aug_rows)):\n            cols_to_noise = rng.choice(len(emb_cols_tr), size=k, replace=False)\n            noise = rng.normal(0.0, sigma[cols_to_noise])\n            emb_values[i, cols_to_noise] = emb_values[i, cols_to_noise] + noise\n        aug_rows[emb_cols_tr] = emb_values.astype(np.float32)\n        X_parts.append(aug_rows)\n        y_parts.append(y_tr.copy())\n        total_added += len(aug_rows)\n    X_tr = pd.concat(X_parts, axis=0, ignore_index=True)\n    y_tr = pd.concat(y_parts, axis=0, ignore_index=True)\n    print(f\"Training augmented mask_rounds={rounds_mask}, noise_rounds={rounds_noise}, added={total_added}, k={k}/{len(emb_cols_tr)}, sigma=0.10, new_train={len(X_tr)}\")\n\nX_tr_scaled = scale_features(X_tr)\nX_val_scaled = scale_features(X_val)\n\n_emb_cols_test = [c for c in X_test.columns if c.startswith('emb_')]\nif len(_emb_cols_test) > 0:\n    arrt = X_test[_emb_cols_test].to_numpy().astype(np.float32)\n    normst = np.linalg.norm(arrt, axis=1, keepdims=True) + 1e-8\n    X_test[_emb_cols_test] = (arrt / normst).astype(np.float32)\nX_test_scaled = scale_features(X_test)\n\ny_tr_scaled = scale_target(y_tr)\ny_val_scaled = scale_target(y_val)\ny_test_scaled = scale_target(y_test)\n\ntrain_w = X_tr.apply(infer_class, axis=1).map(class_weights).to_numpy()\nval_w = X_val.apply(infer_class, axis=1).map(class_weights).to_numpy()\n\nX_tr_arr = X_tr_scaled.values.astype(np.float32)\nX_val_arr = X_val_scaled.values.astype(np.float32)\ny_tr_arr = y_tr_scaled.astype(np.float32)\ny_val_arr = y_val_scaled.astype(np.float32)\nX_test_arr_scaled = X_test_scaled.values.astype(np.float32)\ny_test_arr_scaled = y_test_scaled.astype(np.float32)\n\ninput_dim = X_tr_arr.shape[1]\nprint(\"Prepared arrays (scaled with log1p target). input_dim=\", input_dim)","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:25:38.233949Z","iopub.execute_input":"2025-11-19T12:25:38.234152Z","iopub.status.idle":"2025-11-19T12:25:45.693419Z","shell.execute_reply.started":"2025-11-19T12:25:38.234136Z","shell.execute_reply":"2025-11-19T12:25:45.692328Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class BiomassNet(nn.Module):\n    def __init__(self, input_dim, hidden_units=[4096, 4096, 3584, 3328, 3072, 2816, 2560, 2304, 2048, 1536, 1024, 512, 256], dropout=0.1):\n        super(BiomassNet, self).__init__()\n        self.hidden_units = hidden_units\n        self.fc1 = nn.Linear(input_dim, hidden_units[0])\n        self.bn1 = nn.BatchNorm1d(hidden_units[0])\n        self.fc2 = nn.Linear(hidden_units[0], hidden_units[1])\n        self.bn2 = nn.BatchNorm1d(hidden_units[1])\n        self.fc3 = nn.Linear(hidden_units[1], hidden_units[2])\n        self.bn3 = nn.BatchNorm1d(hidden_units[2])\n        self.fc4 = nn.Linear(hidden_units[2], hidden_units[3])\n        self.bn4 = nn.BatchNorm1d(hidden_units[3])\n        self.fc5 = nn.Linear(hidden_units[3], hidden_units[4])\n        self.bn5 = nn.BatchNorm1d(hidden_units[4])\n        self.fc6 = nn.Linear(hidden_units[4], hidden_units[5])\n        self.bn6 = nn.BatchNorm1d(hidden_units[5])\n        self.fc7 = nn.Linear(hidden_units[5], hidden_units[6])\n        self.bn7 = nn.BatchNorm1d(hidden_units[6])\n        self.fc8 = nn.Linear(hidden_units[6], hidden_units[7])\n        self.bn8 = nn.BatchNorm1d(hidden_units[7])\n        self.fc9 = nn.Linear(hidden_units[7], hidden_units[8])\n        self.bn9 = nn.BatchNorm1d(hidden_units[8])\n        self.fc10 = nn.Linear(hidden_units[8], hidden_units[9])\n        self.bn10 = nn.BatchNorm1d(hidden_units[9])\n        self.fc11 = nn.Linear(hidden_units[9], hidden_units[10])\n        self.bn11 = nn.BatchNorm1d(hidden_units[10])\n        self.fc12 = nn.Linear(hidden_units[10], hidden_units[11])\n        self.bn12 = nn.BatchNorm1d(hidden_units[11])\n        self.fc13 = nn.Linear(hidden_units[11], hidden_units[12])\n        self.bn13 = nn.BatchNorm1d(hidden_units[12])\n        self.fc_out = nn.Linear(hidden_units[12], 1)\n        self.drop = nn.Dropout(p=dropout)\n        self._init_weights()\n\n    def _init_weights(self):\n        layers = [\n            self.fc1, self.fc2, self.fc3, self.fc4, self.fc5, self.fc6,\n            self.fc7, self.fc8, self.fc9, self.fc10, self.fc11, self.fc12,\n            self.fc13, self.fc_out\n        ]\n        for m in layers:\n            nn.init.kaiming_uniform_(m.weight, a=0.0, mode='fan_in', nonlinearity='relu')\n            nn.init.zeros_(m.bias)\n        nn.init.constant_(self.fc_out.bias, 0.1)\n\n    def forward(self, x):\n        x = torch.nn.functional.gelu(self.bn1(self.fc1(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.silu(self.bn2(self.fc2(x)))\n        x = self.drop(x)\n        x = torch.relu(self.bn3(self.fc3(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.gelu(self.bn4(self.fc4(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.silu(self.bn5(self.fc5(x)))\n        x = self.drop(x)\n        x = torch.relu(self.bn6(self.fc6(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.gelu(self.bn7(self.fc7(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.silu(self.bn8(self.fc8(x)))\n        x = self.drop(x)\n        x = torch.relu(self.bn9(self.fc9(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.silu(self.bn10(self.fc10(x)))\n        x = self.drop(x)\n        x = torch.relu(self.bn11(self.fc11(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.gelu(self.bn12(self.fc12(x)))\n        x = self.drop(x)\n        x = torch.nn.functional.silu(self.bn13(self.fc13(x)))\n        x = self.drop(x)\n        x = self.fc_out(x)\n        x = torch.sigmoid(x)\n        return x","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T12:25:45.694362Z","iopub.execute_input":"2025-11-19T12:25:45.694744Z","iopub.status.idle":"2025-11-19T12:25:45.708338Z","shell.execute_reply.started":"2025-11-19T12:25:45.694717Z","shell.execute_reply":"2025-11-19T12:25:45.707563Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class EWCTrainer:\n    def __init__(self, model, device, ewc_lambda=0):\n        self.model = model\n        self.device = device\n        self.ewc_lambda = ewc_lambda\n        self.fisher_matrix = None\n        self.optimal_params = None\n        self.initialized = False\n\n    def compute_fisher_matrix(self, dataloader, sample_size=500):\n        self.model.eval()\n        fisher_dict = {n: torch.zeros_like(p) for n, p in self.model.named_parameters() if p.requires_grad}\n        count = 0\n        for X_batch, y_batch, w_batch in dataloader:\n            if count >= sample_size:\n                break\n            X_batch = X_batch.to(self.device)\n            y_batch = y_batch.to(self.device)\n            w_batch = w_batch.to(self.device)\n            self.model.zero_grad()\n            output = self.model(X_batch)\n            loss = torch.mean(w_batch * (y_batch - output.squeeze()) ** 2)\n            loss.backward()\n            for n, p in self.model.named_parameters():\n                if p.grad is not None:\n                    fisher_dict[n] += p.grad.data ** 2\n            count += len(X_batch)\n        for n in fisher_dict:\n            fisher_dict[n] /= max(count, 1)\n        return fisher_dict\n\n    def ewc_loss(self):\n        if self.ewc_lambda == 0 or self.fisher_matrix is None or self.optimal_params is None:\n            return 0.0\n        loss = 0.0\n        for n, p in self.model.named_parameters():\n            if n in self.fisher_matrix:\n                loss += torch.sum(self.fisher_matrix[n] * (p - self.optimal_params[n]) ** 2)\n        return (self.ewc_lambda / 2.0) * loss\n\n    def update_weights(self, X_data, y_data, base_weights, batch_size=256):\n        self.model.eval()\n        with torch.no_grad():\n            X_tensor = torch.FloatTensor(X_data).to(self.device)\n            preds = []\n            for i in range(0, len(X_tensor), batch_size):\n                batch = X_tensor[i:i+batch_size]\n                pred = self.model(batch).cpu().numpy().flatten()\n                preds.append(pred)\n            preds = np.concatenate(preds)\n        errors = np.abs(preds - y_data)\n        error_75 = np.percentile(errors, 75)\n        error_50 = np.percentile(errors, 50)\n        new_weights = base_weights.copy()\n        hard_mask = errors > error_75\n        medium_mask = (errors > error_50) & (errors <= error_75)\n        new_weights[hard_mask] *= 1.15\n        new_weights[medium_mask] *= 1.05\n        new_weights = np.clip(new_weights, 0.2, 2.0)\n        return new_weights","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T12:25:45.709201Z","iopub.execute_input":"2025-11-19T12:25:45.709736Z","iopub.status.idle":"2025-11-19T12:25:45.733495Z","shell.execute_reply.started":"2025-11-19T12:25:45.709703Z","shell.execute_reply":"2025-11-19T12:25:45.732737Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, train_loader, val_loader, ewc_trainer, X_tr_arr, y_tr_arr, X_val_arr, y_val_arr, base_weights, val_w, epochs=250, lr=0.001):\n    optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=1e-5)\n    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=10, min_lr=1e-7)\n    warmup_epochs = 5\n    best_val_r2 = -float('inf')\n    patience = 50\n    patience_counter = 0\n    current_weights = base_weights.copy()\n    for epoch in range(epochs):\n        if epoch < warmup_epochs:\n            for g in optimizer.param_groups:\n                g['lr'] = lr * float(epoch + 1) / warmup_epochs\n        if epoch == 0 and not ewc_trainer.initialized and ewc_trainer.ewc_lambda > 0:\n            print(\"Initializing Fisher matrix (epoch 0)...\")\n            ewc_trainer.fisher_matrix = ewc_trainer.compute_fisher_matrix(train_loader, sample_size=500)\n            ewc_trainer.optimal_params = {n: p.clone().detach() for n, p in model.named_parameters() if p.requires_grad}\n            ewc_trainer.initialized = True\n        model.train()\n        train_loss = 0.0\n        train_preds_list = []\n        train_labels_list = []\n        train_weights_list = []\n        for X_batch, y_batch, w_batch in train_loader:\n            X_batch = X_batch.to(device)\n            y_batch = y_batch.to(device)\n            w_batch = w_batch.to(device)\n            optimizer.zero_grad()\n            output = model(X_batch)\n            mse_loss = torch.mean(w_batch * (y_batch - output.squeeze()) ** 2)\n            ewc_penalty = ewc_trainer.ewc_loss()\n            loss = mse_loss + ewc_penalty\n            loss.backward()\n            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0)\n            optimizer.step()\n            train_loss += loss.item() * len(X_batch)\n            train_preds_list.append(output.detach().cpu().numpy().flatten())\n            train_labels_list.append(y_batch.cpu().numpy())\n            train_weights_list.append(w_batch.cpu().numpy())\n        train_loss /= len(train_loader.dataset)\n        if train_preds_list:\n            train_preds_all_scaled = np.concatenate(train_preds_list)\n            train_labels_all_scaled = np.concatenate(train_labels_list)\n            train_weights_all = np.concatenate(train_weights_list)\n            train_preds_all = inverse_target(train_preds_all_scaled)\n            train_labels_all = inverse_target(train_labels_all_scaled)\n            train_r2 = weighted_r2(train_preds_all, train_labels_all, train_weights_all)\n        else:\n            train_r2 = 0.0\n        model.eval()\n        with torch.no_grad():\n            val_preds_scaled = []\n            for X_batch, y_batch, w_batch in val_loader:\n                X_batch = X_batch.to(device)\n                output = model(X_batch)\n                val_preds_scaled.append(output.cpu().numpy().flatten())\n            val_preds_scaled = np.concatenate(val_preds_scaled) if len(val_preds_scaled) > 0 else np.array([])\n        val_preds = inverse_target(val_preds_scaled)\n        val_labels = inverse_target(y_val_arr)\n        val_r2 = weighted_r2(val_preds, val_labels, val_w)\n        scheduler.step(val_r2)\n        # Update sample weights to emphasize harder examples next epoch\n        try:\n            new_weights = ewc_trainer.update_weights(X_tr_arr, y_tr_arr, base_weights=current_weights, batch_size=256)\n            current_weights = new_weights\n            train_loader.dataset.tensors = (\n                train_loader.dataset.tensors[0],\n                train_loader.dataset.tensors[1],\n                torch.FloatTensor(new_weights)\n            )\n        except Exception as _:\n            pass\n        if (epoch + 1) % 10 == 0:\n            print(f\"Epoch {epoch+1}/{epochs} - Train Loss: {train_loss:.6f}, Train R2(orig): {train_r2:.6f}, Val R2(orig): {val_r2:.6f}, LR: {optimizer.param_groups[0]['lr']:.2e}\")\n        if val_r2 > best_val_r2:\n            best_val_r2 = val_r2\n            patience_counter = 0\n            best_model_state = model.state_dict().copy()\n        else:\n            patience_counter += 1\n        if patience_counter >= patience:\n            print(f\"Early stopping at epoch {epoch+1}, Best Val R2(orig): {best_val_r2:.6f}\")\n            model.load_state_dict(best_model_state)\n            break\n    return model","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T12:25:45.73433Z","iopub.execute_input":"2025-11-19T12:25:45.734615Z","iopub.status.idle":"2025-11-19T12:25:45.754268Z","shell.execute_reply.started":"2025-11-19T12:25:45.734591Z","shell.execute_reply":"2025-11-19T12:25:45.753539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Creating model and data loaders (scaled inputs / log1p targets)...\")\nmodel = BiomassNet(\n    input_dim,\n    hidden_units=[4096, 4096, 3584, 3328, 3072, 2816, 2560, 2304, 2048, 1536, 1024, 512, 256],\n    dropout=0.1\n).to(device)\n\newc_lambda = 10\nprint(f\"EWC enabled (lambda={ewc_lambda})\")\newc_trainer = EWCTrainer(model, device, ewc_lambda=ewc_lambda)\n\ntrain_dataset = TensorDataset(\n    torch.FloatTensor(X_tr_arr),\n    torch.FloatTensor(y_tr_arr),\n    torch.FloatTensor(train_w)\n)\nval_dataset = TensorDataset(\n    torch.FloatTensor(X_val_arr),\n    torch.FloatTensor(y_val_arr),\n    torch.FloatTensor(val_w)\n)\n\ntrain_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=256, shuffle=False)\n\nprint(\"Starting training (scaled log1p targets; regularization ON, expanded network)...\")\nmodel = train_model(model, train_loader, val_loader, ewc_trainer, X_tr_arr, y_tr_arr, X_val_arr, y_val_arr, train_w, val_w, epochs=250, lr=0.001)\n\nmodel.eval()\nwith torch.no_grad():\n    X_test_tensor = torch.FloatTensor(X_test_arr_scaled).to(device)\n    preds_test_scaled = []\n    for i in range(0, len(X_test_tensor), 256):\n        batch = X_test_tensor[i:i+256]\n        pred = model(batch).cpu().numpy().flatten()\n        preds_test_scaled.append(pred)\n    preds_test_scaled = np.concatenate(preds_test_scaled)\n\npreds_test = inverse_target(preds_test_scaled)\ny_test_orig = inverse_target(y_test_arr_scaled)\n\ntest_w_full = X_test.apply(infer_class, axis=1).map(class_weights).to_numpy()\nw_r2_test = weighted_r2(preds_test, y_test_orig, test_w_full)\nprint(f\"\\nWeighted R2 on test (original scale): {w_r2_test:.6f}\")\n\nprint(\"\\nR2 contribution by target_name (original scale):\")\nr2_breakdown = r2_by_class(preds_test, y_test_orig, X_test)\nfor class_name, metrics in r2_breakdown.items():\n    print(f\"  {class_name}: R2={metrics['r2']:.6f}, Count={metrics['count']}, Weight={metrics['weight']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-19T12:25:45.755103Z","iopub.execute_input":"2025-11-19T12:25:45.755361Z","iopub.status.idle":"2025-11-19T12:29:58.795564Z","shell.execute_reply.started":"2025-11-19T12:25:45.755339Z","shell.execute_reply":"2025-11-19T12:29:58.794634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"DINOV2_DIR = \"/kaggle/input/dinov2/pytorch/giant/1\"\nIMAGES_DIR = \"/kaggle/input/csiro-biomass/test\"\nBATCH = 32\n\ndino_processor = AutoImageProcessor.from_pretrained(DINOV2_DIR, local_files_only=True, use_fast=True)\ndino_model = AutoModel.from_pretrained(DINOV2_DIR, local_files_only=True)\ndino_model.eval().to(device)\n\npaths = list_images(IMAGES_DIR)\nembs = []\nnames = []\n\nfor i in range(0, len(paths), BATCH):\n    batch_paths = paths[i:i+BATCH]\n    imgs = []\n    for p in batch_paths:\n        im = Image.open(p).convert(\"RGB\")\n        imgs.append(im)\n    # Edge-detected versions (RGB)\n    edges = [edge_detect_image(im) for im in imgs]\n\n    inputs_orig = dino_processor(images=imgs, return_tensors=\"pt\")\n    inputs_edge = dino_processor(images=edges, return_tensors=\"pt\")\n    pixel_values_orig = inputs_orig[\"pixel_values\"].to(device)\n    pixel_values_edge = inputs_edge[\"pixel_values\"].to(device)\n\n    with torch.no_grad():\n        with torch.cuda.amp.autocast(enabled=(device.type=='cuda')):\n            outputs_o = dino_model(pixel_values=pixel_values_orig)\n            feats_o = getattr(outputs_o, \"pooler_output\", None)\n            if feats_o is None:\n                feats_o = outputs_o.last_hidden_state[:, 0]\n\n            outputs_e = dino_model(pixel_values=pixel_values_edge)\n            feats_e = getattr(outputs_e, \"pooler_output\", None)\n            if feats_e is None:\n                feats_e = outputs_e.last_hidden_state[:, 0]\n        feats_o = feats_o.detach().cpu().numpy()\n        feats_e = feats_e.detach().cpu().numpy()\n\n    feats = np.concatenate([feats_o, feats_e], axis=1)\n    embs.append(feats)\n    names.extend([os.path.basename(p) for p in batch_paths])\n\nembs = np.concatenate(embs, axis=0)\n# Build combined column names\nhalf = embs.shape[1] // 2\ncols = [f\"emb_{i+1}\" for i in range(half)] + [f\"emb_edge_{i+1}\" for i in range(half)]\n\ntest_emb = pd.DataFrame(embs, columns=cols)\ntest_emb.insert(0, \"filename\", names)\n\ntest_emb.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:29:58.796456Z","iopub.execute_input":"2025-11-19T12:29:58.796748Z","iopub.status.idle":"2025-11-19T12:30:03.688033Z","shell.execute_reply.started":"2025-11-19T12:29:58.796726Z","shell.execute_reply":"2025-11-19T12:30:03.687208Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test['image_filename'] = test['image_path'].map(lambda p: p.split('/')[-1])\ntest = test.merge(test_emb, left_on='image_filename', right_on='filename', how='left')\ntest.drop(columns=['image_filename', 'filename'], inplace=True)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:30:03.688884Z","iopub.execute_input":"2025-11-19T12:30:03.689674Z","iopub.status.idle":"2025-11-19T12:30:03.711499Z","shell.execute_reply.started":"2025-11-19T12:30:03.689654Z","shell.execute_reply":"2025-11-19T12:30:03.710947Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test = pd.get_dummies(test, columns=['target_name'], drop_first=False)\nID = test['sample_id']\ntest = test.drop(columns=['sample_id','image_path','target_name'], errors='ignore')\ntest.head()\n","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:30:03.712207Z","iopub.execute_input":"2025-11-19T12:30:03.712467Z","iopub.status.idle":"2025-11-19T12:30:03.734568Z","shell.execute_reply.started":"2025-11-19T12:30:03.71245Z","shell.execute_reply":"2025-11-19T12:30:03.733839Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# L2-normalize then min-max scale competition test embeddings using train min/max\n_emb_cols_comp = [c for c in test.columns if c.startswith('emb_')]\nif len(_emb_cols_comp) > 0:\n    arrc = test[_emb_cols_comp].to_numpy().astype(np.float32)\n    normsc = np.linalg.norm(arrc, axis=1, keepdims=True) + 1e-8\n    test[_emb_cols_comp] = (arrc / normsc).astype(np.float32)\n\n# Apply train feature scaling\ncomp_minmax_arr = test[feature_columns].to_numpy().astype(np.float32)\nscaled_comp = (comp_minmax_arr - X_min) / range_safe\nscaled_comp = np.clip(scaled_comp, 0.0, 1.0)\nX_comp_scaled = scaled_comp.astype(np.float32)\n\n# Model prediction on scaled features, target will be inverse-transformed afterward\nmodel.eval()\nwith torch.no_grad():\n    test_tensor = torch.FloatTensor(X_comp_scaled).to(device)\n    test_preds_scaled = []\n    for i in range(0, len(test_tensor), 256):\n        batch = test_tensor[i:i+256]\n        pred = model(batch).cpu().numpy().flatten()\n        test_preds_scaled.append(pred)\n    test_preds_scaled = np.concatenate(test_preds_scaled)\n\n# Inverse transform predictions to original target scale\ntest_preds = inverse_target(test_preds_scaled)\ntest_preds","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:30:03.735312Z","iopub.execute_input":"2025-11-19T12:30:03.735593Z","iopub.status.idle":"2025-11-19T12:30:04.089137Z","shell.execute_reply.started":"2025-11-19T12:30:03.735564Z","shell.execute_reply":"2025-11-19T12:30:04.08843Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    'sample_id': ID,\n    'target': test_preds\n})\n\nsubmission_df.to_csv(\"submission.csv\", index=False)\nsubmission_df.head()","metadata":{"execution":{"iopub.status.busy":"2025-11-19T12:30:04.089861Z","iopub.execute_input":"2025-11-19T12:30:04.090097Z","iopub.status.idle":"2025-11-19T12:30:04.103298Z","shell.execute_reply.started":"2025-11-19T12:30:04.09008Z","shell.execute_reply":"2025-11-19T12:30:04.102586Z"},"trusted":true},"outputs":[],"execution_count":null}]}